{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting virtual X frame buffer: Xvfb.\n",
      "env: DISPLAY=:1\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function, division",
    "\n",
    "from IPython.core import display",
    "\n",
    "import matplotlib.pyplot as plt",
    "\n",
    "%matplotlib inline",
    "\n",
    "import numpy as np",
    "\n",
    "\n",
    "# If you are running on a server, launch xvfb to record game videos",
    "\n",
    "# Please make sure you have xvfb installed",
    "\n",
    "import os",
    "\n",
    "if type(os.environ.get(\"DISPLAY\")) is not str or len(os.environ.get(\"DISPLAY\")) == 0:",
    "\n",
    "    !bash ../xvfb start",
    "\n",
    "    os.environ['DISPLAY'] = ':1'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KungFuMaster env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Observation shape: (42, 42, 1)\n",
      "Num actions: 14\n",
      "Action names: ['NOOP', 'UP', 'RIGHT', 'LEFT', 'DOWN', 'DOWNRIGHT', 'DOWNLEFT', 'RIGHTFIRE', 'LEFTFIRE', 'DOWNFIRE', 'UPRIGHTFIRE', 'UPLEFTFIRE', 'DOWNRIGHTFIRE', 'DOWNLEFTFIRE']\n"
     ]
    }
   ],
   "source": [
    "import gym",
    "\n",
    "from atari_util import PreprocessAtari",
    "\n",
    "\n",
    "\n",
    "def make_env():",
    "\n",
    "    env = gym.make(\"KungFuMasterDeterministic-v0\")",
    "\n",
    "    env = PreprocessAtari(env, height=42, width=42,",
    "\n",
    "                          crop=lambda img: img[60:-30, 15:],",
    "\n",
    "                          dim_order='tensorflow',",
    "\n",
    "                          color=False, n_frames=1)",
    "\n",
    "    return env",
    "\n",
    "\n",
    "\n",
    "env = make_env()",
    "\n",
    "\n",
    "obs_shape = env.observation_space.shape",
    "\n",
    "n_actions = env.action_space.n",
    "\n",
    "\n",
    "print(\"Observation shape:\", obs_shape)",
    "\n",
    "print(\"Num actions:\", n_actions)",
    "\n",
    "print(\"Action names:\", env.env.env.get_action_meanings())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANEAAAEICAYAAADBfBG8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFlxJREFUeJzt3XvUHHV9x/H3hyBoASHcEgi3wAGO4CVGxFTKRbyFVAXa\nqsFWUWkJlVA80FMIKFLUAirQKBUImnIRQSqi1BNQCnhpEeRiCJcIJIAQckMIBAVpE7/9Y2Zhstl9\nnnl2dp+Z2f28ztmzszOzu99J5ru/3/xmnu8oIjCzzm1QdgBmdeckMivISWRWkJPIrCAnkVlBTiKz\ngpxEfUjSTpJ+J2lM2bEMAidRAZKmS7pd0u8lrUynPyVJZcYVEY9HxKYRsbbMOAaFk6hDkk4EZgNf\nBsYD44BjgP2AjUoMzUZbRPgxwgewOfB74C+HWe/PgV8Bq4EngNMzy3YBAvhEumwVSRK+FVgAPAuc\n3/R5nwQWpuv+CNi5zfc2PnvD9PVPgC8AtwK/A/4T2Aq4Io3tDmCXzPtnpzGtBu4C9s8sew1waRrD\nQuCfgCWZ5dsD1wBPAY8C/1D2/1fP94eyA6jjA5gKrGnspEOsdxDwBpIW/43ACuCwdFljR78QeDXw\nHuAPwPeBbYEJwErgwHT9w4BFwOuADYHPALe2+d5WSbQI2C39AXgAeAh4V/pZlwH/nnn/36RJtiFw\nIrAceHW67Czgp8BYYIc04ZekyzZIk+40ktZ4V+AR4L1l/5/1dH8oO4A6PtKdbHnTvFvT1uNF4IA2\n7/tX4Lx0urGjT8gsfxr4cOb1NcCn0+nrgaMyyzYAXqBFa9QmiU7NLD8HuD7z+v3A/CG2dxXwpnR6\nnaQA/jaTRG8DHm9676xsgvbjw8dEnXka2FrSho0ZEfH2iNgiXbYBgKS3SbpF0lOSniPprm3d9Fkr\nMtMvtni9aTq9MzBb0rOSngWeAUTSYuWR93uQdKKkhZKeS79r80zc25N09Rqy0zsD2zdiTN97Csnx\nYt9yEnXmF8BLwKHDrPdt4Dpgx4jYnKTr1unI3RPAjIjYIvN4TUTc2uHntSRpf+Ak4EPA2PSH4Tle\niXsZSTeuYcemGB9tinGziJjWzRirxknUgYh4Fvhn4OuS/krSppI2kDQJ2CSz6mbAMxHxB0n7Ah8p\n8LUXArMk7Q0gaXNJHyzwee1sRnK89xSwoaTTgNdmll+dxjFW0gRgZmbZL4HVkk6S9BpJYyS9XtJb\nexBnZTiJOhQRXwJOIBmdWknSPbqI5Fe80Tp8CjhD0vMkB9tXF/i+a4GzgaskrQbuAw7peAPa+xHJ\n8ddDwG9IBjuyXbYzgCUkI2//BXyXpFUmkvNS7wcmpct/C3yDpDvYt5Qe/Jl1RNLfA9Mj4sCyYymL\nWyIbEUnbSdov7b7uSTIEfm3ZcZVpw+FXMVvHRiTd1okkQ/pXAV8vNaKS9aw7J2kqyZnvMcA3IuKs\nnnyRWcl6kkTp1cMPAe8mOQi9AzgiIh7o+peZlaxX3bl9gUUR8QiApKtIzqm0TCJJHt2wKvptRGwz\n3Eq9GliYwLrDoktoOrMu6WhJd0q6s0cxmBX1mzwr9aolanVWfp3WJiLmAHPALZHVW69aoiWseznI\nDsDSHn2XWal6lUR3ALtLmihpI2A6yTVkZn2nJ925iFgjaSbJJSRjgLkRcX8vvsusbJW47MfHRFZR\nd0XEPsOt5Mt+zAqqxWU/xx9/fNkh2ACaPXt2rvXcEpkVVIuWaLTMmDEDgIsuuqjtsqzm9ZrXGely\nqye3RKlWSdJq2UUXXfTyzp+dn03ATpZbfTmJUm4VrFNOohyyCTZjxowhu3btllv/chKZFeSBhZyG\nGyRoXset0eBwS5RDnoRw0gyuWlz2MxonW0c6PJ1nHQ9x19vs2bNzXfbjJDJrI28SuTtnVpCTyKwg\nj85VyNhZY9ebt+rMVSVEYiPhlqgiGgm06sxVLz+y8626nERmBXWcRJJ2TG9gtVDS/ZKOT+efLulJ\nSfPTR1/fm8asyDHRGuDEiLhb0mbAXZJuTJedFxFfKR6eWfV1nEQRsYzkrmlExPOSFpL/1odmfaMr\nx0SSdgHeDNyezpopaYGkuZJaHhm7Auq6sgMJjUd2vlVX4SFuSZvyyl2uV0u6APg8ScXTz5PcqfqT\nze9zBdT1OWHqqVBLJOlVJAl0RUR8DyAiVkTE2oj4I3AxSXF7s75VZHROwDeBhRFxbmb+dpnVDie5\nt6hZ3yrSndsP+Chwr6T56bxTgCPSu2gH8BjgvxGwvlZkdO6/aX33h3mdh2NV5D/hGNrAXjt374NH\nrPP6DXteOaLl3fiMPN9RthkzZrSsMeFEeoUv+7EhOVmG5ySy3IYqbjnInESWm4tOtuYksiE5YYbn\nGgs2rEEdnctbY2FgR+csv0FJmk65O2dWkJPIrCAnkVlBA3NM1HyPoVZn4lstzz5nNc9rfNasWQ/3\nahO64swzdy87hL4zUC3RcAfIeQ6gszfpyvse628DlUTDnfNoXt5q/Tzr2GAZqCRqbkVaLW+ebl6/\n1fvdGg22gUqiZp3c1a75Pa2Ol2yw+IoFszZG7YoFSY8BzwNrgTURsY+kLYHvALuQ/HXrhyLCVTis\nL3WrO/eOiJiUydqTgZsiYnfgpvS1WV/q1XmiQ4GD0ulLgZ8AJ/Xou0ZkJOeDWs1v9Z6sQ37+89HZ\nkA5dv//+ZYfQd7qRRAH8OD2uuSitJzcurZBKRCyTtG0Xvqdrit4m0iyrG925/SJiMnAIcKykA/K8\nqcwKqCM9X9TpOjYYCidRRCxNn1cC15IUa1zRqD+XPq9s8b45EbFPntGPbhvplQvtXvv8kEHxCqib\npHeEQNImwHtIijVeBxyZrnYk8IMi39Ntrc71DLXcbCiFzhNJ2pWk9YHk+OrbEfFFSVsBVwM7AY8D\nH4yIZ4b4HJ8nssoZlfNEEfEI8KYW858G3lnks83qohZXLJiVpH9qLEz+wuSyQ7ABdPdn7s61Xi2S\naNsdKnWayWwdtUiiDa4e6IvNreJqkUTzd5g//EpmJalFEo3faXzZIdgAWsrSXOu5n2RWUC1aIg8s\nWJX5PJFZe7nOE7k7Z1aQk8isoFocE90w2Vcs2Oibene+KxbcEpkV5CQyK8hJZFZQLY6JJs3zFQtW\ngpy7nVsis4I6bokk7UlS5bRhV+A0YAvg74Cn0vmnRMS8jiMEPvLx04ZcPuvE4wA485yvFfmaQhxD\nP8aQb7ftOIki4kFgEoCkMcCTJPUWPgGcFxFf6fSzR2rtSWuTiRKvDnIMgxtDt46J3gksjojfSOrS\nR+Y35uwxycQ5o/7VjsExdC2JpgNXZl7PlPQx4E7gxF4Xsx/EXz/HUJ0YCg8sSNoI+ADwH+msC4Dd\nSLp6y2jze9DNCqhjzh7zyq9PSRzD4MbQjZboEODuiFgB0HgGkHQx8MNWb0prds9J1yt0Ffcg/vo5\nhurE0I0kOoJMV07Sdo1i9sDhJBVRe2oQ++GOoToxFEoiSX8CvBvI1tz9kqRJJHeLeKxpWU8M4q+f\nY6hODEUroL4AbNU076OFIurAIP76OYbqxFCLy36GM4i/fo6hOjH0RRIN4q+fY6hODH2RRIP46+cY\nqhNDXyTRIP76OYbqxNAXSTSIv36OoTox9EUSDeKvn2OoTgx9kUSD+OvnGKoTQy2KNy5fPm20QjF7\n2fjx81y80Ww01KI7d8tk31rFqsstkVlBTiKzgpxEZgXV4pjoHXdPKjsEG0Tjfac8s1FRi5ZouLpz\nZr2Rr+6cWyKzgnIlkaS5klZKui8zb0tJN0p6OH0em86XpK9KWiRpgSTfXMj6Wt6W6BJgatO8k4Gb\nImJ34Kb0NSTVf3ZPH0eTlNAy61u5kigifgY80zT7UODSdPpS4LDM/MsicRuwhaTtuhGsWRUVOSYa\n1yiNlT43rpmdADyRWW9JOm8d3SzeaFamXozOtSrGvd5V2t0s3mhWpiIt0YpGNy19XpnOXwLsmFlv\nByDfWSuzGiqSRNcBR6bTRwI/yMz/WDpKNwV4LlMR1azv5OrOSboSOAjYWtIS4HPAWcDVko4CHgc+\nmK4+D5gGLAJeILlfkVnfypVEEXFEm0XvbLFuAMcWCcqsTnzFgllBTiKzgpxEZgU5icwKchKZFeQk\nMivISWRWkJPIrCAnkVlBTiKzgpxEZgU5icwKchKZFeQkMivISWRWkJPIrCAnkVlBwyZRm+qnX5b0\n67TC6bWStkjn7yLpRUnz08eFvQzerArytESXsH710xuB10fEG4GHgFmZZYsjYlL6OKY7YZpV17BJ\n1Kr6aUT8OCLWpC9vIymLZTaQunFM9Eng+szriZJ+JemnkvZv9yZXQLV+UagCqqRTgTXAFemsZcBO\nEfG0pLcA35e0d0Ssbn5vNyug3nzDlJenD556W5GPqnUMQ6l6fHXWcUsk6UjgfcBfp2WyiIiXIuLp\ndPouYDGwRzcCbSe7c5SlCjGMRN3irbqOkkjSVOAk4AMR8UJm/jaSxqTTu5LcXuWRbgSaVxV2kCrE\nkFW1ePrNsN25NtVPZwEbAzdKArgtHYk7ADhD0hpgLXBMRDTfkqUnGl2UMneYKsTQTpVjq7thk6hN\n9dNvtln3GuCaokF1orFzlNnfr0IMrRw89TYnTw/V4sbHQzl46m187e1nvPz6uFsHM4bhLPjWtJen\nP/0t30i6m3zZj1lBfZFEx9162jrPgxrDUBqtj1uh7qt9dw5gj3sWcBzl7hxlxXD+ua8FYOYJ652K\na7HeVzg/vZf7cOtbfrVvifa4Z8E6z4MUQyOBmqeHWi/P+jYytU+irDITqUoxNJx/7mudLKOgtt25\nquysZcbR6JI1EmW4hGle37qjL1qih970xrJDKDWG7PHNzBNWt3zdnEA+Juqe2rZE1lpzK+NWp/f6\noiWy1i1Lc6s01LrWudon0aB35bKak6MxsJBNJidQ99U+ibIH9mXtzFWIYSjZZLLuq30S2bqcKKOv\n9gMLVfjlr0IMWXvttdd6V5LffMOUyl1d3i/cEpkVVNskWjv3QNbOPXCd12XFUXYMw3Er1Fu1784B\n7Hb82LJDqEQMDQdPvW3d80PnPuBjpR7qtALq6ZKezFQ6nZZZNkvSIkkPSnpvrwJvpQo7chViaOYE\n6q1OK6ACnJepdDoPQNJewHRg7/Q9X28ULum2xbNXsXj2KnY7fiyLZ6/qxVfkjqPsGKxceWos/EzS\nLjk/71Dgqoh4CXhU0iJgX+AXHUeYQxV24irEYOUoMrAwMy1oP1dSow8zAXgis86SdN56ulUBtbHj\nltmNqkIMVp5Ok+gCYDdgEknV03PS+WqxbsvqphExJyL2iYh9OoxhPVXYiasQgy86HV0dJVFErIiI\ntRHxR+Biki4bJC3PjplVdwCWFgvRivCgQu91WgF1u8zLw4HGyN11wHRJG0uaSFIB9ZfFQhxaFX75\nqxCDlafTCqgHSZpE0lV7DJgBEBH3S7oaeICk0P2xEbG2N6FbK+7Kjb6uVkBN1/8i8MUiQeVRlV//\nqsRh5antZT+tVGGIuQox2OhSeleUcoMY5v5EQ133td/yJwH4n/EtR9JHRRViyKpqTfC6ufmGKXfl\nGT2uxbVzJ0xuf+vX2+d9Fkh25LdN+/xohVS5GLJuviF5HurfzYbX+HccTu27c1XYaasQQyvv+5f5\nZYcwEGrRnTMrSf905354yqSyQ7ABlLclr313zqxsTiKzgpxEZgV5YMGsPQ8smBXhgQWzUVKL7tzy\n5dOGWmzWE+PHz+uf7twtk33m3arL3TmzgpxEZgU5icwK6rQC6ncy1U8fkzQ/nb+LpBczyy7sZfBm\nVZBnYOES4HzgssaMiPhwY1rSOcBzmfUXR0RXT+y8426fJ7ISjM9XqKpQBVRJAj4EHDyC0EZs/Ph5\nvfx4s0KKDnHvD6yIiIcz8yZK+hWwGvhMRPy81RslHQ0cnedLrtx++4Jhmo3cEUu71BIN9z3AlZnX\ny4CdIuJpSW8Bvi9p74hYr4JgRMwB5oCvnbN66ziJJG0I/AXwlsa8tJD9S+n0XZIWA3sAhept55E9\nbmqcnG01zzEMXgy9jqPIEPe7gF9HxJLGDEnbNG6lImlXkgqojxQLMb9W/yijfbWDY6h2DL2II88Q\n95Ukt0bZU9ISSUeli6azblcO4ABggaR7gO8Cx0TEM90M2KxqOq2ASkR8vMW8a4BriodlVh++YsGs\noFpcxT0S2f5uWVd/O4ZqxtCrOPoqiapwZYNjGLwYavFHeT7ZamU4YunSXH+UV4skMitJ//xla3L9\na36X/+k/A/DRX3yuF8E4hprF0HkcM3Ot5dE5s4KcRGYFOYnMCqrFMdH47bca1fd1k2OoTgwwsjiW\n5/tLCLdEZkXVoiXaZnz+O3Sfe/ZnOeGkywG4/NLPcsJJo38XO8dQrRiAjuIYyJboikvOYty4TV5+\nPW7cJlxxyVmOYcBj6HUc9WiJtt1iROs3/yON9P3d4BiqE0Ov46jFFQsjuZX8ty85Y53XH/n4aZ0F\nVYBjqG4MI4nj5hum9M9lPyNJIrNuyZtEfXVMZFaGPH8evqOkWyQtlHS/pOPT+VtKulHSw+nz2HS+\nJH1V0iJJCyRN7vVGmJUpT0u0BjgxIl4HTAGOlbQXcDJwU0TsDtyUvgY4hKRAye4kdeUu6HrUZhUy\nbBJFxLKIuDudfh5YCEwADgUuTVe7FDgsnT4UuCwStwFbSNqu65GbVcSIhrjTcsJvBm4HxkXEMkgS\nTdK26WoTgCcyb1uSzlvW9Fm5K6DefMOUkYRpNqpyJ5GkTUkq+Xw6IlYnZbhbr9pi3nqjb66Aav0i\n1+icpFeRJNAVEfG9dPaKRjctfV6Zzl8C7Jh5+w5AzgsozOonz+icgG8CCyPi3Myi64Aj0+kjgR9k\n5n8sHaWbAjzX6PaZ9aWIGPIB/BlJd2wBMD99TAO2IhmVezh93jJdX8C/AYuBe4F9cnxH+OFHBR93\nDrfvRkQ9rlgwK4mvWDAbDU4is4KcRGYFOYnMCqrKH+X9Fvh9+twvtqZ/tqeftgXyb8/OeT6sEqNz\nAJLuzDMSUhf9tD39tC3Q/e1xd86sICeRWUFVSqI5ZQfQZf20Pf20LdDl7anMMZFZXVWpJTKrJSeR\nWUGlJ5GkqZIeTAubnDz8O6pH0mOS7pU0X9Kd6byWhVyqSNJcSSsl3ZeZV9tCNG2253RJT6b/R/Ml\nTcssm5Vuz4OS3jviL8xzqXevHsAYkj+Z2BXYCLgH2KvMmDrcjseArZvmfQk4OZ0+GTi77DiHiP8A\nYDJw33Dxk/wZzPUkf/IyBbi97Phzbs/pwD+2WHevdL/bGJiY7o9jRvJ9ZbdE+wKLIuKRiPhf4CqS\nQif9oF0hl8qJiJ8BzzTNrm0hmjbb086hwFUR8VJEPAosItkvcys7idoVNambAH4s6a60AAs0FXIB\ntm377mpqF3+d/89mpl3QuZnudeHtKTuJchU1qYH9ImIySc29YyUdUHZAPVTX/7MLgN2ASSSVp85J\n5xfenrKTqC+KmkTE0vR5JXAtSXegXSGXuuirQjQRsSIi1kbEH4GLeaXLVnh7yk6iO4DdJU2UtBEw\nnaTQSW1I2kTSZo1p4D3AfbQv5FIXfVWIpum47XCS/yNItme6pI0lTSSp3PvLEX14BUZSpgEPkYyK\nnFp2PB3EvyvJ6M49wP2NbaBNIZcqPoArSbo4/0fyy3xUu/jpoBBNRbbn8jTeBWnibJdZ/9R0ex4E\nDhnp9/myH7OCyu7OmdWek8isICeRWUFOIrOCnERmBTmJzApyEpkV9P8/IlEXoFyAlgAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f151d32b710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEICAYAAACQ6CLfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFyhJREFUeJzt3X20HHV9x/H35z7lhoQACSENhIIPUcEW0xZTqvZIUSwi\nLdiqBVFi5Vg9lh5r7YPaJ2xrq6dF7Dn16EFFUquAjzW1tCUnBVHa8iBGDEINIJiQkIAQyPN9+vaP\nmVv23pmbu3cfZnfz+7zO2XN3fzO78529+92Z+e3M76uIwMzS09fpAMysM5z8Zoly8pslyslvlign\nv1minPxmiXLyJ0zSyZJC0kCnY5kLSRdLurHTcfQ6J38LSbpZ0pOS5lW4zJD03KqWV7WyL6iI+FxE\nvKqTcR0OnPwtIulk4BeBAH61o8F0EWX8OetC/qe0ziXA/wDXAGtqJ0haIulfJD0t6Q5JfyXpWzXT\nXyBpvaQnJP2vpDfUTLtG0sck/auk3ZJuk/ScfNot+WzflbRH0m9MD0pSn6Q/kfSwpJ2S/lHSUdNm\ne6ukbZK2S3pPzXNXS7ozj3uHpI/UTDtD0n9J2iXpu5LOrJl2s6QPSroV2Ae8X9Kd0+J6t6R1+f3X\nSPpOvpwtki6vmXVyHXfl6/gLkt4y7f17Sf6+PpX/fcm0WP5S0q35+3ejpGOnv09JigjfWnAD7gfe\nCfwcMAosq5l2XX47AjgV2AJ8K5+2IH/8m8AA8LPA48AL8+nXAE8Aq/PpnwOuq3ntAJ57iLjemsf2\nbGAh8BXgs/m0k/PnX5vH8dPAY8Ar8+n/Dbw5v78QOCO/fwLwY+Bcsg3I2fnjpfn0m4EfAS/MYz4K\n2A2srInrDuDC/P6Z+bL7gNOAHcAF02IcqHnuW2rev8XAk8Cb82VdlD9eUhPLA8DzgPn54w91+vPS\nDTdv+VtA0suAk4AvRMS3yT5sb8yn9QO/Dvx5ROyLiO8Da2uefh7wUER8JiLGIuIu4MvA62rm+UpE\n3B4RY2TJv2oO4V0MfCQiHoyIPcD7gAundfJ9ICL2RsT3gM+QJRBkX2LPlXRsROyJiP/J298E3BAR\nN0TERESsB+4k+zKYdE1E3JOv01PA1yZfV9JK4AXAOoCIuDkivpe/1t1kX0Yvr3P9XgNsjojP5su6\nFrgP+JWaeT4TET+IiP3AF5jb+3fYcvK3xhrgxoh4PH/8eZ7Z9V9KtkXaUjN/7f2TgJ/Pd593SdpF\nlrA/UTPPozX395Fthet1PPBwzeOH83iWzRDPw/lzAC4l22Lel+9On1cT8+unxfwyYPkMrwnZezL5\npfJG4J8jYh+ApJ+XdJOkxyQ9BbwDqHfXfPr6Ta7DCTWPm3n/Dls99RNPN5I0H3gD0C9p8kM2Dzha\n0ouATcAYsAL4QT79xJqX2AJ8IyLOblOI28iSddJP5vHsyGOajOe+munbACJiM3BR3mH3a8CXJC3J\nY/5sRLztEMudfrnojcCxklaRfQm8u2ba54F/AF4dEQckfZRnkn+2y06nr9/kOvz7LM9Lnrf8zbsA\nGCc7ll+V304BvglcEhHjZMfZl0s6QtILyDoHJ30deJ6kN0sazG8vlnRKncvfQXY8P5NrgXdLepak\nhcBfA9fnhxCT/jSP7YVkfQ/XA0h6k6SlETEB7MrnHQf+CfgVSb8sqV/SsKQzJa1gBvnyvgT8Ldlx\n+vqayUcCT+SJv5r8kCn3GDBxiHW8gez9e6OkgbzT81Sy99UOwcnfvDVkx5Q/iohHJ29kW7KL82Pr\ny8g6vR4FPkuWkAcBImI38CrgQrKt2KPAh8n2HupxObA23/1+Q8n0q/Nl3gL8EDgA/M60eb5B1im4\nAfi7iJg8geYc4B5Je4C/J+ugOxARW4DzgfeTJecW4A+Y/fP0eeCVwBenffm8E/gLSbuBPyM7Lgcg\nPzT4IHBrvo5n1L5gRPyYrN/kPWSdjn8InFdzCGYzUN4jahWS9GHgJyJizawzm7WJt/wVyH/HPy07\n30WryTrSvtrpuCxt7vCrxpFku/rHAzuBK8h++jLrGO/2myXKu/1miWpqt1/SOWS9wP3ApyLiQ4ea\nf3BoQQwPH9PMIs3sEA4ceJLRkb2qZ96Gkz8/bfVjZOd1bwXukLQuP3211PDwMZy++rJGF2lms7jz\n9n+oe95mdvtXA/fn54yPkF24cn4Tr2dmFWom+U9g6vnbW5l6PjUAkn4rvyz0ztHRvU0szsxaqZnk\nLzuuKPx0EBFXRcTpEXH64OCCJhZnZq3UTIffVqZeoLKC/IKQmWjPfgZv3dTEIs3sUHRwf93zNrPl\nvwNYmV8wMkR2bvq6Jl7PzCrU8JY/IsYkXQb8B9lPfVdHxD0ti8zM2qqp3/kj4gaySyrNrMf4DD+z\nRFV6YU8sms+BXzytykWaJSW+eXPd83rLb5YoJ79Zopz8Zoly8pslyslvlqhKe/tHFgdbLhybfcYO\niyhetiAVRzyqdz5LT6c+GyP31L8Mb/nNEuXkN0uUk98sUU5+s0RVO25/iJiY2hESB/srDaFh88YL\nTTFSjL3rRkIfnChvnygZi2W8rnEfq9Nf8mb2zfAGj3bZdqwszpL/Rcs//2X/1xl02TtmZlVx8psl\nyslvlignv1mimq3Y8xCwGxgHxiLi9EPOPyoGHpladl7FfrSuFP3Ft6onYp/p672s36zbOivn0v/Y\nbbGXKftfzNAf2yiN1v+mtaK3/5ci4vEWvI6ZVci7/WaJajb5A7hR0rcl/VbZDLUVe8b3umKPWbdo\ndrf/pRGxTdJxwHpJ90XELbUzRMRVwFUAwytO7IUjM7MkNDt097b8705JXyUr3nnLjPMDMW1foyc6\nzSjGDb0Re8xwAllp7F321Vz2ns/UCahuu1K8JM6Jkv9FX4s7/Oai4d1+SQskHTl5H3gV4FpcZj2i\nmS3/MuCrkiZf5/MR8e8ticrM2q6Zcl0PAi9qYSxmViH/1GeWqEov6RWgDnZwNKOvBzr3ynRdR9gc\nlHZKdtlVxzMquxq5yz5D3vKbJcrJb5YoJ79Zopz8Zoly8pslqtLe/p4+vbfk1Mxe+OVixtN7y2Lv\nttN7y2Lv4dN7u+0z5C2/WaKc/GaJcvKbJcrJb5aoaiv2DARjS6b1zHRZJ9OMyjqaeiH2w20QTDuk\nGHSJbjObhZPfLFFOfrNEOfnNEjVrh5+kq4HzgJ0R8VN522LgeuBk4CHgDRHx5KxLmxDa7+8bs7Zp\ncYnua4BzprW9F9gQESuBDfljM+shsyZ/Pg7/E9OazwfW5vfXAhe0OC4za7NG98GXRcR2gPzvcTPN\nOKViz549DS7OzFqt7QfgEXFVRJweEaf3L1zY7sWZWZ0aPcNvh6TlEbFd0nJgZ13PCugb6ZURGM16\n0BzO0mx0y78OWJPfXwN8rcHXMbMOmTX5JV0L/DfwfElbJV0KfAg4W9Jm4Oz8sZn1kFl3+yPiohkm\nvaLFsZhZhXzGjVmiKr+kd3zJaKWLNEvKgC/pNbNZOPnNEuXkN0uUk98sUdWW6B4RQ1uHqlykWVI0\nhzNoveU3S5ST3yxRTn6zRDn5zRLl5DdLlJPfLFFOfrNEOfnNEuXkN0tUPSP5XC1pp6RNNW2XS3pE\n0sb8dm57wzSzVmu0aAfAlRGxKr/d0NqwzKzdGi3aYWY9rplj/ssk3Z0fFhzTsojMrBKNJv/HgecA\nq4DtwBUzzTilYs/evQ0uzsxaraHkj4gdETEeERPAJ4HVh5j3mYo9CxY0GqeZtVhDyZ9X6Zn0WmDT\nTPOaWXeadTCPvGjHmcCxkrYCfw6cKWkVWXGgh4C3tzFGM2uDRot2fLoNsZhZhXyGn1minPxmiXLy\nmyXKyW+WKCe/WaKc/GaJcvKbJcrJb5YoJ79Zopz8Zoly8pslyslvlignv1minPxmiXLymyXKyW+W\nKCe/WaLqqdhzoqSbJN0r6R5J78rbF0taL2lz/tfDd5v1kHq2/GPAeyLiFOAM4LclnQq8F9gQESuB\nDfljM+sR9VTs2R4Rd+X3dwP3AicA5wNr89nWAhe0K0gza705HfNLOhn4GeA2YFlEbIfsCwI4bobn\nuGiHWReqO/klLQS+DPxuRDxd7/NctMOsO9WV/JIGyRL/cxHxlbx5x2TxjvzvzvaEaGbtUE9vv8jG\n6b83Ij5SM2kdsCa/vwb4WuvDM7N2mbVoB/BS4M3A9yRtzNveD3wI+IKkS4EfAa9vT4hm1g71VOz5\nFqAZJr+iteGYWVV8hp9Zopz8Zomq55i/ZTQBA/umHkGMz4/CfDHTQUYHDewvBjUxVJxvor+4Pr2i\nf6S+N358qHfX0Z7hLb9Zopz8Zoly8pslyslvlqhKO/wGH93Lir/5rylt237/JYX5Ro7qbIfS0NPF\njq/jr7it0LbrTauLbSvbElLLqeQtPnH9nkLbwM7iZRwPXnJ8oa2XOzpT5S2/WaKc/GaJcvKbJcrJ\nb5YoJ79Zoqo9vXfePPpPfs7UtokqI6hP32ixbWDZ0kLbRH8FwbSJJoq/aIwcPa/Q1v902TnMJS/Y\nw+9FqrzlN0uUk98sUU5+s0Q1U7HnckmPSNqY385tf7hm1ir1dPhNVuy5S9KRwLclrc+nXRkRf1fv\nwg4sHeAHb5s6vH//wZLTQjt8pujBxcUANr/rWYU2jZc8uYMdmGVjI8w4OMKB4rwPXVAy78CiQtPQ\no933P7O5q2cMv+3AZHGO3ZImK/aYWQ9rpmIPwGWS7pZ09UyFOl2xx6w7NVOx5+PAc4BVZHsGV5Q9\nzxV7zLpTwxV7ImJHRIxHxATwSaB4fauZda1Zj/lnqtgjaflkoU7gtcCm2V6rbxTm75zaqTR6ZPcN\n4KnxYgDDjxXnGz2y2NbJ69pPW/1Aoe2Yof2l837jgeLAA1eu/mKhbWl/8Xr+S9a9s9A2sLsLR121\nQ2qmYs9FklaR9fM+BLy9LRGaWVs0U7HnhtaHY2ZV8Rl+Zoly8pslqtJLegH6xqY+Lru0NDo8GKTG\n6mwrO8Ovg5e2btz47ELbH59dXjn9bb/wjULbfSPLC22b9q8otPXVWdnHupu3/GaJcvKbJcrJb5Yo\nJ79Zoqrt8BNMDE5t6vTZfPWaHjd0X+yDTxW/yz+2+eWl8951+vWFtkfH9xXa/vr75xTa+g42EJx1\nHW/5zRLl5DdLlJPfLFFOfrNEOfnNElVpb3/0w8hR007d7cKKPePDxdOLx+eXzNgDg1buvXtxafvz\n9l1SaBsbKX4cBh8uVvGxw4O3/GaJcvKbJcrJb5aoeir2DEu6XdJ384o9H8jbnyXpNkmbJV0vqaSc\nq5l1q3o6/A4CZ0XEnnwU329J+jfg98gq9lwn6RPApWTDec9Iw+MMnjJ1QMjRe4sVYTrdCTi6qNiT\nd+RJTxXa9t5/VKGtf393nfPbN1Yez/OX7yi03bfjuEJb4A6/w9WsW/7I7MkfDua3AM4CvpS3rwUu\naEuEZtYW9Y7b35+P3LsTWA88AOyKiMnxbbYyQwmv2oo9Y08XLxwxs86oK/nz4hyrgBVkxTlOKZtt\nhuf+f8WegUVHNB6pmbXUnHr7I2IXcDNwBnC0pMk+gxXAttaGZmbtVE/FnqXAaETskjQfeCXwYeAm\n4HXAdcAaoHykyBoRYny8+39dLOu0G58oibvbLugvMTFYfhriK469r9B2/4+PLbSNtjwi6xb19PYv\nB9ZK6ifbU/hCRHxd0veB6yT9FfAdspJeZtYj6qnYczdZWe7p7Q/i4pxmPav798HNrC2c/GaJqvSS\n3qGBMU5a8sSUtgf7FxbmUxde5rts0e5C2w/nFWPvP9Bd36djC8o7/H56eEuhbe/jxZ9ifc724au7\nPqlmVhknv1minPxmiXLymyWq0g6/sYk+du6Z1knWhZ1740cWg+pTseOsb7QHzvBbWFJbHLjnYPE6\nLO3vYH1xq5y3/GaJcvKbJcrJb5YoJ79Zoqot2rFngJFbl0xpGy7vj+qooaeLHV/btp5YaDtipIpo\nmjN/Z/k5ev96xRmFtkVnF7cFZaXJrXtpvP55veU3S5ST3yxRTn6zRDn5zRLVTMWeayT9UNLG/Laq\n/eGaWas0U7EH4A8i4kuHeO4UfaOwYHsP1LU+jGii/P3e8/xjCm3DTxRPa46+7j+F2Z7RN4dfz+oZ\nwy+Asoo9ZtbDGqrYExG35ZM+KOluSVdKKi3qNqViz4G9LQrbzJrVUMUeST8FvA94AfBiYDHwRzM8\n95mKPcMLWhS2mTWr0Yo950TE9ryI50HgM3gYb7Oe0nDFHknLI2K7JJFV6N0022uFYNwjQlasvMNu\nbNjX7h+O5lJEqpmKPf+ZfzEI2Ai8o4FYzaxDmqnYc1ZbIjKzSvgMP7NEOfnNElXp9fwTg7Bvmc8Y\nM2uXuYy/4C2/WaKc/GaJcvKbJcrJb5aoSjv8+kbhiB2+INCsXfpG5zBv+8Iws27m5DdLlJPfLFFO\nfrNEVdrhp5hbRREzm5uSSvIz8pbfLFFOfrNEOfnNEuXkN0tU3cmfD9/9HUlfzx8/S9JtkjZLul6S\nR+cz6yFz6e1/F3AvsCh//GHgyoi4TtIngEuBjx/qBfrGggU7ppYU6RspqRLTX37N/9CTB+cQbmuN\nLSx+t8VAMc6JktiHdo0U2jReXO92mBgqH6hzbGHxwu+ywR8HdxfPF+0bqeYnm+gvbptGji7fxvSN\nF7u5NVZsG9hT/F9UZWTxcKFNYyWf/4HybfLQrpLP/7SKTP0H6v9c1Vu0YwXwGuBT+WMBZwGTpbrW\nko3ga2Y9ot7d/o8CfwhMfq0sAXZFxORmfCtwQtkTayv2jI64Yo9Zt6inSu95wM6I+HZtc8mspacX\n1FbsGRxyxR6zblHPMf9LgV+VdC4wTHbM/1HgaEkD+dZ/BbCtfWGaWavVM27/+8jq8iHpTOD3I+Ji\nSV8EXgdcB6wBvjbba40cJX507tSdjeEdxY6n8eHycxRXbOjcWAA7XlysQzq6qBjP+Lxi2/HfLD53\ncHc1nWZ7VpR3kP34tJL3smR/bsnG4kdk4SPVdJqNLip2Vm57WfnOat9IMfh5TxTbjrur+bga9cjL\ni+/lwL5ijKMLyz/nJ5Z8/qd3as6lpHozv/P/EfB7ku4n6wP4dBOvZWYVm9OFPRFxM1mhTiLiQVyc\n06xn+Qw/s0Q5+c0SpYjqOtEkPQY8nD88Fni8soW31+G0LuD16XaHWp+TImJpPS9SafJPWbB0Z0Sc\n3pGFt9jhtC7g9el2rVof7/abJcrJb5aoTib/VR1cdqsdTusCXp9u15L16dgxv5l1lnf7zRLl5DdL\nVOXJL+kcSf8r6X5J7616+c2SdLWknZI21bQtlrQ+H9JsvaRjOhnjXEg6UdJNku6VdI+kd+XtPbdO\nkoYl3S7pu/m6fCBv7+kh59o1hF6lyS+pH/gY8GrgVOAiSadWGUMLXAOcM63tvcCGiFgJbMgf94ox\n4D0RcQpwBvDb+f+kF9fpIHBWRLwIWAWcI+kMnhlybiXwJNmQc71kcgi9SS1Zn6q3/KuB+yPiwYgY\nIbsc+PyKY2hKRNwCPDGt+Xyyocygx4Y0i4jtEXFXfn832YfsBHpwnSKzJ384mN+CHh5yrp1D6FWd\n/CcAW2oezzj8V49ZFhHbIUsm4LgOx9MQSScDPwPcRo+uU76LvBHYCawHHqDOIee6VMND6M2m6uSv\ne/gvq5akhcCXgd+NiKc7HU+jImI8IlaRjS61GjilbLZqo2pMs0PozabSQp1k31In1jw+XIb/2iFp\neURsl7ScbKvTMyQNkiX+5yLiK3lzT69TROySdDNZP0avDjnX1iH0qt7y3wGszHsrh4ALgXUVx9AO\n68iGMoM6hzTrFvkx5KeBeyPiIzWTem6dJC2VdHR+fz7wSrI+jJvIhpyDHlkXyIbQi4gVEXEyWa78\nZ0RcTKvWJyIqvQHnAj8gOxb746qX34L4rwW2A6NkezKXkh2HbQA2538XdzrOOazPy8h2G+8GNua3\nc3txnYDTgO/k67IJ+LO8/dnA7cD9wBeBeZ2OtYF1OxP4eivXx6f3miXKZ/iZJcrJb5YoJ79Zopz8\nZoly8pslyslvlignv1mi/g+aB6t2OkdUTAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f154a156be0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "s = env.reset()",
    "\n",
    "for _ in range(100):",
    "\n",
    "    s, _, _, _ = env.step(env.action_space.sample())",
    "\n",
    "\n",
    "plt.title('Game image')",
    "\n",
    "plt.imshow(env.render('rgb_array'))",
    "\n",
    "plt.show()",
    "\n",
    "\n",
    "plt.title('Agent observation')",
    "\n",
    "plt.imshow(s.reshape([42, 42]))",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple agent for fully-observable MDP\n",
    "\n",
    "Here's a code for an agent that only uses feedforward layers. Please read it carefully: you'll have to extend it later!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf",
    "\n",
    "tf.reset_default_graph()",
    "\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Conv2D, Dense, Flatten",
    "\n",
    "\n",
    "\n",
    "class FeedforwardAgent:",
    "\n",
    "    def __init__(self, name, obs_shape, n_actions, reuse=False):",
    "\n",
    "        \"\"\"A simple actor-critic agent\"\"\"",
    "\n",
    "\n",
    "        with tf.variable_scope(name, reuse=reuse):",
    "\n",
    "            # Note: number of units/filters is arbitrary, you can and should change it at your will",
    "\n",
    "            self.conv0 = Conv2D(32, (3, 3), strides=(2, 2), activation='elu')",
    "\n",
    "            self.conv1 = Conv2D(32, (3, 3), strides=(2, 2), activation='elu')",
    "\n",
    "            self.conv2 = Conv2D(32, (3, 3), strides=(2, 2), activation='elu')",
    "\n",
    "            self.flatten = Flatten()",
    "\n",
    "            self.hid = Dense(128, activation='elu')",
    "\n",
    "            self.logits = Dense(n_actions)",
    "\n",
    "            self.state_value = Dense(1)",
    "\n",
    "\n",
    "            # prepare a graph for agent step",
    "\n",
    "            _initial_state = self.get_initial_state(1)",
    "\n",
    "            self.prev_state_placeholders = [tf.placeholder(m.dtype,",
    "\n",
    "                                                           [None] + [m.shape[i] for i in range(1, m.ndim)])",
    "\n",
    "                                            for m in _initial_state]",
    "\n",
    "            self.obs_t = tf.placeholder('float32', [None, ] + list(obs_shape))",
    "\n",
    "            self.next_state, self.agent_outputs = self.symbolic_step(",
    "\n",
    "                self.prev_state_placeholders, self.obs_t)",
    "\n",
    "\n",
    "    def symbolic_step(self, prev_state, obs_t):",
    "\n",
    "        \"\"\"Takes agent's previous step and observation, returns next state and whatever it needs to learn (tf tensors)\"\"\"",
    "\n",
    "\n",
    "        nn = self.conv0(obs_t)",
    "\n",
    "        nn = self.conv1(nn)",
    "\n",
    "        nn = self.conv2(nn)",
    "\n",
    "        nn = self.flatten(nn)",
    "\n",
    "        nn = self.hid(nn)",
    "\n",
    "        logits = self.logits(nn)",
    "\n",
    "        state_value = self.state_value(nn)",
    "\n",
    "\n",
    "        # feedforward agent has no state",
    "\n",
    "        new_state = []",
    "\n",
    "\n",
    "        return new_state, (logits, state_value)",
    "\n",
    "\n",
    "    def get_initial_state(self, batch_size):",
    "\n",
    "        \"\"\"Return a list of agent memory states at game start. Each state is a np array of shape [batch_size, ...]\"\"\"",
    "\n",
    "        # feedforward agent has no state",
    "\n",
    "        return []",
    "\n",
    "\n",
    "    def step(self, prev_state, obs_t):",
    "\n",
    "        \"\"\"Same as symbolic state except it operates on numpy arrays\"\"\"",
    "\n",
    "        sess = tf.get_default_session()",
    "\n",
    "        feed_dict = {self.obs_t: obs_t}",
    "\n",
    "        for state_ph, state_value in zip(self.prev_state_placeholders, prev_state):",
    "\n",
    "            feed_dict[state_ph] = state_value",
    "\n",
    "        return sess.run([self.next_state, self.agent_outputs], feed_dict)",
    "\n",
    "\n",
    "    def sample_actions(self, agent_outputs):",
    "\n",
    "        \"\"\"pick actions given numeric agent outputs (np arrays)\"\"\"",
    "\n",
    "        logits, state_values = agent_outputs",
    "\n",
    "        policy = np.exp(logits) / np.sum(np.exp(logits),",
    "\n",
    "                                         axis=-1, keepdims=True)",
    "\n",
    "        return [np.random.choice(len(p), p=p) for p in policy]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_parallel_games = 5",
    "\n",
    "gamma = 0.99",
    "\n",
    "\n",
    "agent = FeedforwardAgent(\"agent\", obs_shape, n_actions)",
    "\n",
    "\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "action logits:\n",
      " [[ 0.22320291 -0.09882765 -0.29847801 -0.25794494  0.26606399  0.14439808\n",
      "   0.01381227  0.16320689 -0.01212743 -0.05400544  0.09274483 -0.01097699\n",
      "  -0.05752631 -0.08983701]]\n",
      "state values:\n",
      " [[-0.05059339]]\n"
     ]
    }
   ],
   "source": [
    "state = [env.reset()]",
    "\n",
    "_, (logits, value) = agent.step(agent.get_initial_state(1), state)",
    "\n",
    "print(\"action logits:\\n\", logits)",
    "\n",
    "print(\"state values:\\n\", value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's play!\n",
    "Let's build a function that measures agent's average reward."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(agent, env, n_games=1):",
    "\n",
    "    \"\"\"Plays an entire game start to end, returns session rewards.\"\"\"",
    "\n",
    "\n",
    "    game_rewards = []",
    "\n",
    "    for _ in range(n_games):",
    "\n",
    "        # initial observation and memory",
    "\n",
    "        observation = env.reset()",
    "\n",
    "        prev_memories = agent.get_initial_state(1)",
    "\n",
    "\n",
    "        total_reward = 0",
    "\n",
    "        while True:",
    "\n",
    "            new_memories, readouts = agent.step(",
    "\n",
    "                prev_memories, observation[None, ...])",
    "\n",
    "            action = agent.sample_actions(readouts)",
    "\n",
    "\n",
    "            observation, reward, done, info = env.step(action[0])",
    "\n",
    "\n",
    "            total_reward += reward",
    "\n",
    "            prev_memories = new_memories",
    "\n",
    "            if done:",
    "\n",
    "                break",
    "\n",
    "\n",
    "        game_rewards.append(total_reward)",
    "\n",
    "    return game_rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[500.0, 900.0, 600.0]\n"
     ]
    }
   ],
   "source": [
    "env_monitor = gym.wrappers.Monitor(env, directory=\"kungfu_videos\", force=True)",
    "\n",
    "rw = evaluate(agent, env_monitor, n_games=3,)",
    "\n",
    "env_monitor.close()",
    "\n",
    "print(rw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<video width=\"640\" height=\"480\" controls>\n",
       "  <source src=\"./kungfu_videos/openaigym.video.1.25110.video000000.mp4\" type=\"video/mp4\">\n",
       "</video>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show video",
    "\n",
    "from IPython.display import HTML",
    "\n",
    "import os",
    "\n",
    "\n",
    "video_names = list(filter(lambda s: s.endswith(",
    "\n",
    "    \".mp4\"), os.listdir(\"./kungfu_videos/\")))",
    "\n",
    "\n",
    "HTML(\"\"\"",
    "\n",
    "<video width=\"640\" height=\"480\" controls>",
    "\n",
    "  <source src=\"{}\" type=\"video/mp4\">",
    "\n",
    "</video>",
    "\n",
    "\"\"\".format(\"./kungfu_videos/\"+video_names[-1]))  # this may or may not be _last_ video. Try other indices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training on parallel games\n",
    "\n",
    "We introduce a class called EnvPool - it's a tool that handles multiple environments for you. Here's how it works:\n",
    "![img](https://s7.postimg.cc/4y36s2b2z/env_pool.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from env_pool import EnvPool",
    "\n",
    "pool = EnvPool(agent, make_env, n_parallel_games)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for each of n_parallel_games, take 10 steps",
    "\n",
    "rollout_obs, rollout_actions, rollout_rewards, rollout_mask = pool.interact(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Actions shape: (5, 10)\n",
      "Rewards shape: (5, 10)\n",
      "Mask shape: (5, 10)\n",
      "Observations shape:  (5, 10, 42, 42, 1)\n"
     ]
    }
   ],
   "source": [
    "print(\"Actions shape:\", rollout_actions.shape)",
    "\n",
    "print(\"Rewards shape:\", rollout_rewards.shape)",
    "\n",
    "print(\"Mask shape:\", rollout_mask.shape)",
    "\n",
    "print(\"Observations shape: \", rollout_obs.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Actor-critic\n",
    "\n",
    "Here we define a loss function that uses rollout above to train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "observations_ph = tf.placeholder('float32', [None, None, ] + list(obs_shape))",
    "\n",
    "actions_ph = tf.placeholder('int32', (None, None,))",
    "\n",
    "rewards_ph = tf.placeholder('float32', (None, None,))",
    "\n",
    "mask_ph = tf.placeholder('float32', (None, None,))",
    "\n",
    "\n",
    "initial_memory_ph = agent.prev_state_placeholders",
    "\n",
    "dummy_outputs = agent.symbolic_step(",
    "\n",
    "    initial_memory_ph, observations_ph[:, 0])[1]",
    "\n",
    "\n",
    "_, outputs_seq = tf.scan(",
    "\n",
    "    lambda stack, obs_t: agent.symbolic_step(stack[0], obs_t),",
    "\n",
    "    initializer=(initial_memory_ph, dummy_outputs),",
    "\n",
    "    # [time, batch, h, w, c]",
    "\n",
    "    elems=tf.transpose(observations_ph, [1, 0, 2, 3, 4])",
    "\n",
    ")",
    "\n",
    "\n",
    "# from [time, batch] back to [batch, time]",
    "\n",
    "outputs_seq = [tf.transpose(",
    "\n",
    "    tensor, [1, 0] + list(range(2, tensor.shape.ndims))) for tensor in outputs_seq]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# actor-critic losses",
    "\n",
    "# logits shape: [batch, time, n_actions], states shape: [batch, time, n_actions]",
    "\n",
    "logits_seq, state_values_seq = outputs_seq",
    "\n",
    "\n",
    "logprobs_seq = tf.nn.log_softmax(logits_seq)",
    "\n",
    "logp_actions = tf.reduce_sum(",
    "\n",
    "    logprobs_seq * tf.one_hot(actions_ph, n_actions), axis=-1)[:, :-1]",
    "\n",
    "\n",
    "current_rewards = rewards_ph[:, :-1] / 100.",
    "\n",
    "current_state_values = state_values_seq[:, :-1, 0]",
    "\n",
    "next_state_values = state_values_seq[:, 1:, 0] * mask_ph[:, :-1]",
    "\n",
    "\n",
    "# policy gradient",
    "\n",
    "# compute 1-step advantage using current_rewards, current_state_values and next_state_values",
    "\n",
    "advantage =  # YOUR CODE",
    "\n",
    "assert advantage.shape.ndims == 2",
    "\n",
    "# compute policy entropy given logits_seq. Mind the sign!",
    "\n",
    "entropy =  # YOUR CODE",
    "\n",
    "assert entropy.shape.ndims == 2",
    "\n",
    "\n",
    "actor_loss = - tf.reduce_mean(logp_actions *",
    "\n",
    "                              tf.stop_gradient(advantage)) - 1e-2 * tf.reduce_mean(entropy)",
    "\n",
    "\n",
    "# compute target qvalues using temporal difference",
    "\n",
    "target_qvalues =  # YOUR CODE",
    "\n",
    "critic_loss = tf.reduce_mean(",
    "\n",
    "    (current_state_values - tf.stop_gradient(target_qvalues))**2)",
    "\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(1e-5).minimize(actor_loss + critic_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train \n",
    "\n",
    "just run train step and see if agent learns any better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_batch(rollout_length=10):",
    "\n",
    "    prev_mem = pool.prev_memory_states",
    "\n",
    "    rollout_obs, rollout_actions, rollout_rewards, rollout_mask = pool.interact(",
    "\n",
    "        rollout_length)",
    "\n",
    "\n",
    "    feed_dict = {",
    "\n",
    "        observations_ph: rollout_obs,",
    "\n",
    "        actions_ph: rollout_actions,",
    "\n",
    "        rewards_ph: rollout_rewards,",
    "\n",
    "        mask_ph: rollout_mask,",
    "\n",
    "    }",
    "\n",
    "    for placeholder, value in zip(initial_memory_ph, prev_mem):",
    "\n",
    "        feed_dict[placeholder] = value",
    "\n",
    "    return feed_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output",
    "\n",
    "from tqdm import trange",
    "\n",
    "from pandas import DataFrame",
    "\n",
    "moving_average = lambda x, **kw: DataFrame(",
    "\n",
    "    {'x': np.asarray(x)}).x.ewm(**kw).mean().values",
    "\n",
    "\n",
    "rewards_history = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlcVPe9+P/Xh2ETBRFFRAbFBaMigkDcErOYaFySmEbJ\n0iymN439fbO0vWnTLDe3adPkNm2atOm9TZs9ms3EJWqiidEYq4m4gIL7ggqyKQiKILLO5/fHOSgq\nCsowh5l5Px+PeTDzOWdm3gfxvM/y+XzeSmuNEEII7+NjdQBCCCGsIQlACCG8lCQAIYTwUpIAhBDC\nS0kCEEIILyUJQAghvJQkACGE8FKSAIQQwktJAhBCCC/la3UAF9OjRw8dExNjdRhCCOFWMjIyjmqt\nw1tar0MngJiYGNLT060OQwgh3IpSKrc168klICGE8FKSAIQQwktJAhBCCC/Voe8BNKeuro78/Hyq\nq6utDkVYLDAwELvdjp+fn9WhCOGW3C4B5OfnExwcTExMDEopq8MRFtFaU1paSn5+Pv369bM6HCHc\nkttdAqqurqZ79+6y8/dySim6d+8uZ4JCtIHbJQBAdv4CkL8DIdrKLROAEMJ7ZBdXsHLnEavD8EiS\nANxQTEwMR48etToMIVziqQXb+P8+zKC0ssbqUDyOJIA20lrjcDja7fPr6+vb7bOF6Oi2F5STnnuM\neodmUWah1eF4HEkAlyEnJ4chQ4bw8MMPk5SUxAcffMCYMWNISkoiNTWVyspKNm7cyO233w7A4sWL\n6dSpE7W1tVRXV9O/f38A3nrrLa688koSEhKYPn06VVVVADzwwAM8/vjjXH/99Tz55JOUlpYyceJE\nRowYwc9+9jO01gCcPHmSqVOnkpCQwLBhw/j000+t+YUI0U4+SMulk5+NKyKCmZeed/pvXziH23UD\nber3X+xgZ+EJp37m0N4hPHdLXIvr7dmzh/fee4/nn3+e22+/nZUrV9K5c2f+9Kc/8eqrr/LMM8+w\nZcsWANauXcuwYcPYtGkT9fX1jBo1CoDbb7+dhx56CIBnn32Wd955h8ceewyAvXv3snLlSmw2Gz//\n+c+5+uqr+e1vf8vSpUt58803Afj666/p3bs3S5cuBaC8vNypvwshrHS8qpZFmQVMT7YzJDKE/160\nnR2FJxgW1dXq0DyGnAFcpr59+zJ69GjWr1/Pzp07ueqqq0hMTGT27Nnk5ubi6+vLwIED2bVrFxs3\nbuTxxx9nzZo1rF27lnHjxgGwfft2xo0bR3x8PB999BE7duw4/fmpqanYbDYA1qxZw7333gvA1KlT\n6datGwDx8fGsXLmSJ598krVr19K1q/zHEJ7js/Q8auod3D+mL7cO742/rw+fpedZHZZHceszgNYc\nqbeXzp07A8Y9gAkTJvDJJ5+ct864ceP46quv8PPz48Ybb+SBBx6goaGBv/zlL4BxqWfRokUkJCTw\n/vvvs3r16vM+v1FzXR4HDRpERkYGy5Yt4+mnn2bixIn89re/deJWCmGNBofmg/W5jOoXxuBeIQDc\nFNeLxZmFPDNlCIF+Nosj9AxyBtBGo0eP5ocffiA7OxuAqqoq9u7dC8A111zD3/72N8aMGUN4eDil\npaXs3r2buDgjcVVUVBAZGUldXR0fffTRBb/jmmuuOb38q6++4tixYwAUFhYSFBTEvffey69//Ws2\nb97cnpsqhMus3lNMXtkpZo6NOd12R4qd8lN1rNwlXUKdxa3PADqC8PBw3n//fe6++25qaoxuai+8\n8AKDBg1i1KhRHDlyhGuuuQaA4cOH07Nnz9NH83/4wx8YNWoUffv2JT4+noqKima/47nnnuPuu+8m\nKSmJa6+9lj59+gCwbds2nnjiCXx8fPDz8+Of//ynC7ZYiPY3Oy2XXiGBTBgacbpt7IAe9O4ayLz0\nfG4e3tvC6DyH6sh31VNSUvS5BWF27drFkCFDLIpIdDTy9+B5DpRUMv6Vf/OrCYN47IbYs5a98s0e\n/u+7bNY9NZ7Irp0sirDjU0plaK1TWlpPLgEJITqUD9bn4mdT3DWyz3nLZiTb0RoWbi6wIDLPIwlA\nCNFhnKypZ356PlPjIwkPDjhved/unRnVL0zGBDiJJAAhRIfx+ZYCKmrqub/Jzd9zpaZEk1NaRXru\nMdcF5qEkAQghOgStNXPScoiP6sqI6NALrjclvhed/W18tknGBLRVqxOAUsqmlNqilPrSfN1PKbVB\nKbVPKfWpUsrfbA8wX2eby2OafMbTZvsepdRNzt4YIYT7Wn+gjL1HKrl/TN+LTvUd5O/L1OGRLN1W\nxMkamSurLS7lDOAXwK4mr/8E/FVrHQscAx402x8EjmmtBwJ/NddDKTUUuAuIAyYBryulZDSHEAKA\nOWk5dAvy45aElrt4pqZEU1XbwLJtRe0fmAdrVQJQStmBqcDb5msFjAfmm6vMBm4zn08zX2Muv8Fc\nfxowV2tdo7U+CGQDI52xEUII91Z4/BTf7DzCnVf2adUo35S+3ejXozPzMvJdEJ3nau0ZwN+A3wCN\n8x53B45rrRvPv/KBKPN5FJAHYC4vN9c/3d7Me05TSs1SSqUrpdJLSkouYVO8h7vUA3A4HLz99ttc\nffXVJCQkMGHCBL788suz1pk3bx5xcXH4+Phw7piPP/7xjwwcOJArrriC5cuXuzJ04WIfbziE1pp7\nRp3f9bM5SilmJNvZeLCMnKMn2zk6z9ViAlBK3QwUa60zmjY3s6puYdnF3nOmQes3tdYpWuuU8PDw\nlsKzVHvXAgD3rQegteaee+5h+/btLFiwgKysLN5//30+/PBDXnvttdPrDRs2jIULF54eLd1o586d\nzJ07lx07dvD111/z8MMP09DQ4OrNEC5QU9/AJxsPccOQCKLDglr9vulJdnwUzJezgMvWmqkgrgJu\nVUpNAQKBEIwzglCllK95lG8HGqs15APRQL5SyhfoCpQ1aW/U9D2X56un4PC2Nn3EeXrFw+SXLrg4\nJyeHyZMnc/3115OWlsaiRYvYs2cPzz33HDU1NQwYMID33nuPnTt38tJLL7Fw4UIWL17MXXfdRXl5\nOQ6Hg6FDh3LgwAHeeust3nzzTWpraxk4cCAffPABQUFBPPDAA4SFhbFlyxaSkpJ45plnuPvuuykp\nKWHkyJFn1QO44447yM/Pp6Ghgf/+7//mzjvvPCve/fv388gjj1BSUkJQUBBvvfUWsbGxxMbGsn//\nfsrLywkLC2P16tVcc801jBs3jvfee48PP/yQgwcPUlRUxN69e3n11VdZv349X331FVFRUXzxxRf4\n+fnx/PPP88UXX3Dq1CnGjh3LG2+8gVKK2bNn07dvX1566czvMioqio8//pibbrqJGTNmEBUVdcFR\nvI2/s4CAAPr168fAgQPZuHEjY8aMccI/suhIlm0rovRkLTPHxFzS+3p1DWRcbDgLNufznxMGYfOR\nGtGXqsUzAK3101pru9Y6BuMm7iqt9T3Ad8AMc7WZwGLz+RLzNebyVdrYYy0B7jJ7CfUDYoGNTtsS\nF9qzZw/3338/W7ZsoXPnzrzwwgusXLmSzZs3k5KSwquvvkpSUlKz9QA2bNhwVj2ATZs2kZWVxZAh\nQ3jnnXdOf0djPYBXXnmF3//+91x99dVs2bKFW2+9lUOHDgFn6gFkZWWxfft2Jk2adF6ss2bN4n//\n93/JyMjgL3/5Cw8//DA2m41Bgwaxc+dOvv/+e5KTk1m7di01NTXk5+czcOBAwEgeS5cuZfHixdx7\n771cf/31bNu2jU6dOp2uQfDoo4+yadMmtm/fzqlTp05f4pkzZw7PPPMMJSUlTJkyhbFjx/LEE08w\nb948HnnkkRaL1xQUFBAdfeZ4wW63U1Agoz890ex1ufQP78xVA7tf8nvvSImmqLyaH7I7/iXRjqgt\nk8E9CcxVSr0AbAEa917vAB8opbIxjvzvAtBa71BKfQbsBOqBR7TWbTunv8iRentqrAUAnFUPAKC2\ntpYxY8ZcsB5AQ0PDWfUAnn32WY4fP05lZSU33XSmZ+y59QAWLlwInF8P4Ne//jVPPvkkN9988+nP\nbVRZWcm6detITU093dY4Yd24ceNYs2YNBw8e5Omnn+att97i2muv5corrzy97uTJk/Hz8yM+Pp6G\nhobTCSY+Pp6cnBwAvvvuO/785z9TVVVFWVkZcXFx3HLLLdTX1xMSEsJ//ud/MmvWLG655RZmzJhB\nXFwcw4cPZ8WKFRf9HTc3yvNiXQOFe8rKO05m3nF+f2vcZf373ji0J6FBfszLyOeaQR37knFHdEkJ\nQGu9GlhtPj9AM714tNbVQOq57eayF4EXLzXIjqbpXP0duR6Aw+EgNDSUzMzMZmP717/+RWFhIc8/\n/zwvv/zy6ctAjQICjKH4jbONNsbg4+NDfX091dXVPPzww6SnpxMdHc3vfvc7qqurAU4nr927d/PH\nP/4Rm83GxIkTASguLqZnz54X+Q0bR/x5eWf6DOTn59O7t8wA6WnmpOXS2d/G7Unn9QdplQBfG9MS\nevPJpjzKq+roGuTn5Ag9m4wEbqOOXA8gJCSEfv36MW/ePMBIVllZWQCMGjWKdevW4ePjQ2BgIImJ\nibzxxhvnnUVcTOPOvkePHlRWVjJ//vyzlldUVHDFFVfwzTff4HA4WLFiBdXV1bzyyivn3as41623\n3srcuXOpqanh4MGD7Nu3j5EjpdewJymtrOGLrYVMT7YTHHj5O+7UlGhq6x0syZJLhJdKEkAbNa0H\nMHz4cEaPHs3u3bsBmq0HMHz48PPqAUyYMIHBgwdf8Duee+451qxZQ1JSEt98881Z9QBGjhxJYmIi\nL774Is8+++x57/3oo4945513SEhIIC4ujsWLjVs1AQEBREdHn76UNW7cOCoqKoiPj2/1toeGhvLQ\nQw8RHx/Pbbfddtblo7vvvpvf/va3PP3007z++utcffXVxMbGMnfuXB555JHT2/v5559jt9tJS0tj\n6tSppy+DxcXFcccddzB06FAmTZrEP/7xj9NnFcIzfJqeR61Z8rEt4nqHMCQyRMYEXAapByDahcPh\nYPr06SQmJvL4448THBxMSUkJCxcu5MEHH8TX1zm1iOTvwT3VNzi49uXVxPQI4qOfjm7z5737/UGe\n/3InX/9y3OkSkt5M6gEIS/n4+DB//nzCwsK46aabSEpK4ic/+QmxsbFO2/kL9/Xt7mIKjp/i/kvs\n+nkht42Iws+mmJcuZwGXwi3/J2qtpUeIG7DZbDz22GM89thj7fL5HfnsVVzcnLQcokI7ccPgi3cG\naK2wzv7cMDiCRVsKeGryYPxscmzbGm73WwoMDKS0tFT+83s5rTWlpaUEBgZaHYq4RNnFFfyQXco9\no/vg68QddWqKndKTtazaXey0z/R0bncGYLfbyc/PR+YJEoGBgdjtdqvDEJdo9rpc/H19uDMluuWV\nL8G1g8IJDw5gXno+N8X1cupneyq3SwB+fn7069fP6jCEEJfhRHUdCzbnc8vw3nTvcn7Jx7bwtflw\ne1IUb689SHFFNT2D5eywJW53CUgI4b4WZuRTVdvAAxcp+dgWqcnRNDg0i7bImIDWkAQghHAJh0Mz\nJy2XEX1Cibd3bZfvGNizCyP6hDIvPV/uE7aCJAAhhEv8sP8oB46evORZPy/VHSnR7CuuJCu/vF2/\nxxNIAhBCuMTsdbn06OLP5Pj2vUF78/BIAv18mJcuReNbIglACNHu8sqq+Hb3Ee4e2YcA3/ad0iM4\n0I/JwyJZklVIdZ0UEboYSQBCiHb34YZcfJTix60s+dhWqcl2KqrrWb7jsEu+z11JAhBCtKvqugY+\n3ZTHTXERRHbt5JLvHN2/O/ZunWRqiBZIAhBCtKslWYUcr6pz2rw/reHjYxSN/2H/UfKPVbnse92N\nJAAhRLvRWjN7XQ5XRAQzql+YS797epIdrWFBhowJuJAWE4BSKlAptVEplaWU2qGU+r3Z/r5S6qBS\nKtN8JJrtSin1d6VUtlJqq1IqqclnzVRK7TMfMy/0nUIIz7D50HF2FJ7g/rF9XT6BY3RYEGMHdGf+\n5jwcDhkT0JzWnAHUAOO11glAIjBJKdU4gfcTWutE89FYd3AyRsH3WGAW8E8ApVQY8BwwCqOU5HNK\nqW7O2xQhREczJy2H4EBfbku8vJKPbXVHSjR5ZafYcLDMku/v6FpMANpQab70Mx8XS6fTgDnm+9YD\noUqpSOAmYIXWukxrfQxYAUxqW/hCiI6quKKaZduKSE2OpnOANdOO3RTXi+AAX+ZlyJiA5rTqHoBS\nyqaUygSKMXbiG8xFL5qXef6qlGqc2SkKaPrbzjfbLtQuhPBAczfmUdegua+NJR/bopO/jZsTerNs\nWxEV1XWWxdFRtSoBaK0btNaJgB0YqZQaBjwNDAauBMKAJ83Vm7vQpy/Sfhal1CylVLpSKl2mfBbC\nPdU1OPhoQy7XDgqnX4/OlsaSmmKnus7B0q1FlsbREV1SLyCt9XFgNTBJa11kXuapAd7DuK4PxpF9\n04m+7UDhRdrP/Y43tdYpWuuU8PDwSwlPCNFBfLPjCEdO1DBzrHVH/41GRIcyILyzFI1vRmt6AYUr\npULN552AG4Hd5nV9lHFr/zZgu/mWJcD9Zm+g0UC51roIWA5MVEp1M2/+TjTbhBAeZnZaDn3Cgrh2\nkHNKPraFUoo7UqLJyD3G/pLKlt/gRVpzBhAJfKeU2gpswrgH8CXwkVJqG7AN6AG8YK6/DDgAZANv\nAQ8DaK3LgD+Yn7EJeN5sE0J4kF1FJ9h4sIz7RvfF5tMxanf/KCkKm49ivpwFnKXFW/Na663AiGba\nx19gfQ08coFl7wLvXmKMQgg3Mictl0A/H1JTOk65zp7BgVw3KJwFGfn8asIgp9YidmfyWxBCOE15\nVR2LthRwW2IUoUH+VodzltQUO8UVNazdd9TqUDoMSQBCCKeZl5HHqboGS7t+Xsj4wRGEdfaXMQFN\nSAIQQjiFw6H5YH0uV8Z0I653+5R8bAt/Xx9uS4xixc4jlJ2stTqcDkESgBDCKf69r4Tc0iqXzvp5\nqVJT7NQ1aBZnygRxIAlACOEkc9bl0DM4gJvi2rfkY1sMiQxhWFSI1AkwSQIQQrRZztGTrN5bwo9H\n9cHft2PvVu5IiWZn0Ql2FErR+I79LyWEcAsfrs/FphQ/Humako9tcWtCb/xtPnIWgCQAIUQbVdXW\n81l6HpPjI+kZEmh1OC0KDfJnQlwEizILqKn37qLxkgCEEG2yOLOQE9X1zOyAXT8vJDXZzvGqOr7d\nVWx1KJaSBCCEuGyNJR+HRoaQ3Nd96juNiw2nV0gg89K9e0yAJAAhxGXbeLCM3YcrmGlByce2sPko\npidH8e+9JRw5UW11OJaRBCCEuGxz0nLp2smPWxPcr7bTjORoHBoWbvbeMQGSAIQQl+VweTVf7zjM\nnVdG08nfZnU4l6xfj85cGdONeel5GHNYeh9JAEKIy/LxhlwcWnPvKPe5+Xuu1ORoDhw9yeZDx6wO\nxRKSAIQQl6ymvoGPNx7ihsE96dM9yOpwLtuU4ZEE+du8dkyAJAAhxCX7evthjlbWduh5f1qjS4Av\nU+Ij+XJrEVW19VaH43KSAIQQl2z2uhz69+jM1QN7WB1Km6Um26msqeerbYetDsXlJAEIIS7Jtvxy\nNh86zn1j+uLTQUo+tsXIfmH07R7klXUCWlMUPlAptVEplaWU2qGU+r3Z3k8ptUEptU8p9alSyt9s\nDzBfZ5vLY5p81tNm+x6l1E3ttVFCiPYzJy2HIH8b05M7TsnHtlBKkZpsZ/2BMg6VVlkdjku15gyg\nBhivtU4AEoFJSqnRwJ+Av2qtY4FjwIPm+g8Cx7TWA4G/muuhlBoK3AXEAZOA15VS7td3TAgvduxk\nLYuzCrk9KYqQQD+rw3Ga25PsKAXzN3vXzeAWE4A2VJov/cyHBsYD88322cBt5vNp5mvM5TcoY4jg\nNGCu1rpGa30QyAZGOmUrhBAu8Wl6HrX1Dre/+Xuu3qGduHpgDxZk5ONweM+YgFbdA1BK2ZRSmUAx\nsALYDxzXWjfeNs8HGocCRgF5AObycqB70/Zm3iOE6OAaHJoP0nIZ0787gyKCrQ7H6VJToik4fop1\n+0utDsVlWpUAtNYNWutEwI5x1D6kudXMn83dFdIXaT+LUmqWUipdKZVeUlLSmvCEEC6wancxBcdP\nMXOs+w78upiJQyMICfT1qpvBl9QLSGt9HFgNjAZClVK+5iI7UGg+zweiAczlXYGypu3NvKfpd7yp\ntU7RWqeEh4dfSnhCiHY0Jy2HyK6B3DgkwupQ2kWgn41piVF8vf0w5afqrA7HJVrTCyhcKRVqPu8E\n3AjsAr4DZpirzQQWm8+XmK8xl6/SxkQbS4C7zF5C/YBYYKOzNkQI0X6yiytZu+8o947ui6/Nc3uP\np6bYqal38OXW845NPVJr/iUjge+UUluBTcAKrfWXwJPA40qpbIxr/O+Y678DdDfbHweeAtBa7wA+\nA3YCXwOPaK29uxyPEG7iw/W5+Nt8uPPK6JZXdmPxUV25IiKYz7xkagjfllbQWm8FRjTTfoBmevFo\nrauB1At81ovAi5cephDCKpU19czPyOfm4ZH06BJgdTjtSilFaoqdF5buYt+RCmI98GZ3U557LieE\ncIrPN+dTWVPP/WNjrA7FJW4bEYWvj2JehuefBUgCEEJckNaa2Wm5JNi7khgdanU4LtGjSwDjB/dk\n4eYC6hocVofTriQBCCEuKG1/KdnFlR438KslqSnRHK2sYfUez+6KLglACHFBs9NyCOvsz9ThkVaH\n4lLXXRFOjy7+Hl80XhKAEKJZBcdPsWLnEe66MppAP++atsvP5sPtSXZW7S7maGWN1eG0G0kAQohm\nfbQ+F4B7RnvmyN+WpCbbqXdoFm3x3KLxkgCEEOeprmtg7qY8JgyNICq0k9XhWCI2IpiE6FDmped7\nbNF4SQBCiPMs3VpE2claZnrZzd9zpSbb2XOkgm0F5VaH0i4kAQghzjMnLYeBPbswZkB3q0Ox1C0J\nvQnw9fHYovGSAIQQZ8nMO05Wfjkzx/TFKOXhvbp28mPSsF4sziygus7zZq6RBCCEOMucdTl0CfDl\nR0meUfKxrVKTozlRXc+KnUesDsXpJAEIIU47WlnDl1uLmJFsp0tAi1OFeYWxA7oTFdqJzzxwTIAk\nACHEaZ9uyqO2wcG9Xtr1szk+PorpSVF8n32UwuOnrA7HqSQBCCEAqG9w8OH6XMbF9mBgzy5Wh9Oh\nzEiORmtY6GFF4yUBCCEAWLHzCEXl1V43709r9OkexOj+YczL8KwxAZIAhBCAMe9PVGgnxg/uaXUo\nHVJqcjS5pVVsPFhmdShOIwlACMGewxWsP1DGfWP6YvPx7q6fFzI5vhddAnw9qk6AJAAhBHPScgjw\n9eHOFM8u+dgWQf6+3Dw8kmXbijhZU291OE7RmqLw0Uqp75RSu5RSO5RSvzDbf6eUKlBKZZqPKU3e\n87RSKlsptUcpdVOT9klmW7ZS6qn22SQhxKUoP1XHws0FTEvsTbfO/laH06Glptipqm1g6bYiq0Nx\nitacAdQDv9JaDwFGA48opYaay/6qtU40H8sAzGV3AXHAJOB1pZRNKWUD/gFMBoYCdzf5HCGERRZk\n5HOqrkFu/rZCUp9u9A/v7DF1AlpMAFrrIq31ZvN5BbALiLrIW6YBc7XWNVrrg0A2RvH4kUC21vqA\n1roWmGuuK4SwiMOh+WB9Lsl9uzEsqqvV4XR4SilmJNvZlHOMg0dPWh1Om13SPQClVAwwAthgNj2q\nlNqqlHpXKdXNbIsCmqbHfLPtQu3nfscspVS6Uiq9pMSzy7EJYbW12Uc5ePQk94+RgV+tNT3Jjo+C\n+RnufxbQ6gSglOoCLAB+qbU+AfwTGAAkAkXAK42rNvN2fZH2sxu0flNrnaK1TgkPD29teEKIyzBn\nXQ49ugQweZh3lXxsi4iQQK4dFM6CjAIaHO49JqBVCUAp5Yex8/9Ia70QQGt9RGvdoLV2AG9hXOIB\n48i+aVcCO1B4kXYhhAUOlVaxak8xPx7VB39f6RB4KVJTojl8oprvs49aHUqbtKYXkALeAXZprV9t\n0t70kOFHwHbz+RLgLqVUgFKqHxALbAQ2AbFKqX5KKX+MG8VLnLMZQohL9eGGXGxKcc+oPlaH4nZu\nGNKT0CA/t58grjXT/V0F3AdsU0plmm3PYPTiScS4jJMD/AxAa71DKfUZsBOjB9EjWusGAKXUo8By\nwAa8q7Xe4cRtEUK00qnaBj7dlMdNw3oRERJodThuJ8DXxm2JUXy84RDHq2oJDXLP7rMtJgCt9fc0\nf/1+2UXe8yLwYjPtyy72PiGEayzJKqD8VJ3Xl3xsi9QUO++vy2FJVqHbdqGVC39CeBmtNbPX5TK4\nVzBXxnRr+Q2iWXG9uzI0MsSty0VKAhDCy2TkHmNn0Qlmjo3x+pKPbZWaYmdbQTm7ik5YHcplkQQg\nhJeZnZZLSKAv0xJ7Wx2K25uWGIWfTbntWYAkACG8SPGJar7aVsQdKdEE+UvJx7YK6+zPhKERLMos\noLbeYXU4l0wSgBBe5OONh2jQWko+OlFqcjRlJ2tZtbvY6lAumSQAIbxEbb2DjzYc4rpB4cT06Gx1\nOB5jXGwPegYHuOUEcZIAhPASy3ccpqSihvvHxlgdikfxtflwe5Kd1XtLKK6otjqcSyIJQAgvMSct\nh77dg7g2VubYcrbUFDsNDs3nmwusDuWSSAIQwgvsKCxnU84x7hvdFx8p+eh0A8K7kNy3m9sVjZcE\nIIQX+CAtl05+NlKTpeRje0lNtpNdXElm3nGrQ2k1SQBCeLjjVbUsyizgthFRdA3yszocjzV1eCSB\nfj585kZjAiQBCOHh5qXnU13nkKIv7Sw40I8pwyL5MquQU7UNVofTKpIAhPBgDWbJx5H9whgSGWJ1\nOJdOa9jwBnzyY1j1Auz6Ao7lGu0dUGpKNBU19SzfcdjqUFpFhgIK4cH+vbeYQ2VVPDlpsNWhXLpT\nx2DRI7BnKXSNhr1fgzaPrANDITLh7EfYAPCx9ph2VL8wosM6MS8jj9tGXKx0escgCUAIDzZ7XS4R\nIQFMjIvQzZKnAAAd80lEQVSwOpRLU7gFPpsJJwph0p9g1M+gvhqO7ISiTDi8FYqyYMO/oKHWeI9/\nF+gVD72Gn0kK4VeAzXX3PXx8FDOSovnryr3klVURHRbksu++HJIAhPBQB4+e5N97S3h8wiD8bG5y\ntVdr2PQ2LH8GukTAf3wN9hRjmV8nsCcbj0YNdVCyG4rMhFCUBVs+hI1vGMttARAx9OwzhZ5x4Nd+\nRXCmJ0fxt2/3smBzPr+8cVC7fY8zSAIQwkPNScvBz6a4a6SbdP2sqYAvfgHbF0DsRPjRGxAUdvH3\n2PzMo/54GHGP0eZogLIDZkLINH7uWAQZ7xvLlQ3CB5sJwTxb6BUPAcFO2Qx7tyCuGtCD+Rn5/Hx8\nbIced9FiAlBKRQNzgF6AA3hTa/2aUioM+BSIwSgJeYfW+phZQ/g1YApQBTygtd5sftZM4Fnzo1/Q\nWs927uYIIQBO1tQzPz2fKfGR9Ax2g5KPR3bAZ/cbO+4bnoOrfnn51/N9bNAj1njEzzDatIbjh86c\nJRzeCtkrIetj800Kug84+/JRZELLCegCUlPs/GJuJusPljJ2QI/L2w4XaM0ZQD3wK631ZqVUMJCh\nlFoBPAB8q7V+SSn1FPAU8CQwGaMQfCwwCvgnMMpMGM8BKRh1hDOUUku01secvVFCeLvPtxRQUVPv\nHqUKt3wES38FgV1h5hcQc7Xzv0Mp6NbXeAy99Ux7xeEml48yIT8ddiw8s7xr9NkJoddwCO5lfN5F\n3BTXi+BAX+al57t3AtBaFwFF5vMKpdQuIAqYBlxnrjYbWI2RAKYBc7QxHnq9UipUKRVprrtCa10G\nYCaRScAnTtweIbye1po5aTkMiwohqU+o1eFcWG0VLHsCMj+EftfC9LehS0/XxhDcy3gMmnimrars\nzE3moiwjQexeinHcCnTuefblo8gECO17VlII9LNxS0JvFm7O5/fT4ggJ7JgD8C7pHoBSKgYYAWwA\nIszkgNa6SCnV+C8XBTSdFzXfbLtQuxDCidYfKGPvkUr+PGN4xy35eHSf0cuneCdc+6Tx8LFZHZUh\nKAz6X2c8GtVUwOHtZy4fFWXB/lVNuqV2bXL5KBEiE7gjqTcfbzjE0q1F3D2yj+u3oxVanQCUUl2A\nBcAvtdYnLvKH1dwCfZH2c79nFjALoE+fjvlLE6Ijm5OWQ2iQH7cmdNCSj9sXwJKfg28A3LsABt5g\ndUQtCwiGvmOMR6O6aiOBnT5TyIKNb0FDDQAJfkEs7dyH3O9iwW+SkSDCB4Ovv0Ubcb5WJQCllB/G\nzv8jrXXjBbIjSqlI8+g/Emgsh5MPNO12YAcKzfbrzmlffe53aa3fBN4ESElJ6ZjD/YTooAqPn+Kb\nnUf46bh+BPp1kCPqRvU1RvfOTW9D9GiY8S50deOLAH6BEJVkPBo11MHRvVCUhSrKImz3BvoeXwGL\nvzCW2/yh59Aml5ASISLO6OJqgdb0AlLAO8AurfWrTRYtAWYCL5k/Fzdpf1QpNRfjJnC5mSSWA/+j\nlOpmrjcReNo5myGEAPh4wyG01tw7qoPN+3Msx7jkU5QJY38ON/zWpQO0XMbmZ+zQI+Ig8cfYrq4m\n8Y8reSLFj5/FVpy5fLRrCWw2O0EqG/QYdM7N5ngIbP+pO1pzBnAVcB+wTSmVabY9g7Hj/0wp9SBw\nCEg1ly3D6AKajdEN9CcAWusypdQfgE3mes833hAWQrRdTX0Dn2w8xA1DIjrWCNTdS2HR/zOe3/UJ\nDJ5ibTwu1DM4kOuuiODtXeU8OO12fJt2Sy3PO3sA24HVsHXumTf3vw7uX9zMpzpPa3oBfU/z1+8B\nzrt4Z/b+eeQCn/Uu8O6lBCiEaJ1l24ooPVnLzI7S9bOhDlb+DtL+D3qPgNT3oVuMxUG5XmpKNCt3\nFbNmXwnjB5tTcigFoX2Mx5Cbz6xcccQ8S8gE3/a/LCQjgYXwELPX5dI/vDNXDexudShQXgDzfwJ5\nG2DkLJj4gnHT1wuNH9yT7p39mZeefyYBXEhwBARPgNgJLonNTSYIEUJcTFbecTLzjjNzTIz1XT+z\nv4U3xhmje2e8C1Ne9tqdP4CfzYfbRkSxctcRyk7WWh3OWSQBCOEB5qTl0tnfxu1JFvaqcTTAqhfh\nw+nQpRfM+jcMm25dPB1IaoqdugbNoi0dq2i8JAAh3FxpZQ1fbC1kerKdYKtGnFYcgTnTYM2fjUnZ\nfroSegy0JpYOaHCvEIbbuzIvo2OVi5QEIISb+zQ9j9p6C0s+5nxvXPLJT4dpr8O0f4B/B+qF1EGk\nJtvZVXSC7QXlVodymiQAIdxYfYODj9Yf4qqB3RnY0znTGbeawwFrX4HZt0BACDy06syUzOI8tyZE\n4e/rw7z0vJZXdhFJAEK4sW93F1Nw/JTrZ/2sKoNP7oRvn4e4H8Gs74zCK+KCugb5MXFoBIuzCqmp\n7xhF4yUBCOHG5qTlEBXaiRsGu3AWzbxN8K9xxsClqa/A9HecVkzF092REs3xqjpW7ixueWUXkAQg\nhJvKLq7gh+xS7hndB19XlHzUGtJeh/cmGTN3PvgNXPnTFufGF2dcNbAHkV0DmZfRMS4DSQIQwk3N\nScvF39eHO1NcUPKxuhw+uw+WPw2DJsHP1hije8UlsfkopifZWbO3hMPl1VaHIwlACHdUUV3Hgox8\nbhnem+5d2nmQVVEWvHEt7PkKJr4Id34InTpwoZkObkayHYeGBZut7xIqCUAIN7RwcwEnaxuYObYd\nu35qDenvwtsToKEWHlgGYx+VSz5tFNOjMyP7hTE/Ix9j6jTrSAIQws1orZmdlkNidCjD7e10JF5T\nCQtnwZf/Cf3Gwc/WQp9R7fNdXig12c7BoyfJyLW2JLokACHczA/ZpRwoOdl+R//Fu+Gt8bB9Pox/\nFn48Dzp3gAnmPMiU+EiC/G3MS7f2MpAkACHczOy0HLp39mdKfKTzPzxrLrx1PZw6ZsxFf80T4CO7\nCWfrHODL1PhIvtxaSFVtvWVxyHTQwm3VNTjYcug49Q0Oq0NxmYqaer7ddYSHrxtIgK8TSz7WnYKv\nfgOb50Dfq2HGOxDcy3mfL86TmhLNvIx8lm07zIxkuyUxSAIQbsfh0HyxtZBXV+wlt7TK6nBczs+m\n+PGoPs77wNL9RrnGI9tg3K/huqfBJruG9nZlTDdiugcxLz1PEoAQLdFa8+2uYv7yzR52H65gSGQI\n//fjEYS3dzfIDqZHcAC9Q51ULWrHIlj8qFHL9p75LitEIkApRWpKNC8v30Nu6Un6du/s8hhaUxT+\nXeBmoFhrPcxs+x3wEFBirvaM1nqZuexp4EGgAfi51nq52T4JeA2wAW9rrV9y7qYIT7b+QCkvL99D\nRu4xYroH8fe7R3BzfCQ+PtIl8bLU18KK/4YN/wL7lUa5xq7WHIV6s9uTonjlmz3Mz8jnVxOvcPn3\nt+YM4H3g/4A557T/VWv9l6YNSqmhwF1AHNAbWKmUGmQu/gcwAcgHNimllmitd7YhduEFtuWX8/I3\ne1izt4SIkAD+50fxpKbY8XPF1Aee6vghmPcAFGTA6Efgxt+Br7/FQXmnyK6duDo2nAUZ+fzyxkHY\nXHxA05qi8GuUUjGt/LxpwFytdQ1wUCmVDYw0l2VrrQ8AKKXmmutKAhDNyi6u5NUVe1i27TChQX48\nM2Uw94+JIdDPiTc+vdGer+HznxmDvO78EIbcYnVEXu+OFDuPfryFdfuPMi423KXf3ZZ7AI8qpe4H\n0oFfaa2PAVHA+ibr5JttAHnntMuoEnGeguOneG3lXuZn5BPoZ+Pn4wfy02v6E2JVpStP0VAPq/4A\nP/wNeg2HO2ZDWH+roxLAjUMi6NrJj3np+W6TAP4J/AHQ5s9XgP8Amjt/0TQ/3qDZMdBKqVnALIA+\nfZzY00F0aEcra/jHd9l8tP4QAA+M7cfD1w+gh5fd4G0XJwph/oNwaB2k/Afc9EfwC7Q6KmEK9LMx\nLbE3czflUV5VR9cg1x3sXFYC0FofaXyulHoL+NJ8mQ80nZrQDhSazy/Ufu5nvwm8CZCSkmLtRBmi\n3Z2oruPtNQd45/uDnKprIDU5mp/fGEuUs3q5eLv938GCnxr9/G9/G4anWh2RaEZqcjRz0nJZsrWQ\n+0a7rrTnZSUApVSk1rrIfPkjYLv5fAnwsVLqVYybwLHARowzg1ilVD+gAONG8Y/bErhwb9V1DcxJ\ny+H11fs5XlXH1PhIHp84iAHhXawOzTM4GmDNy7D6JQgfDHfMgfBBLb9PWGJYVAiDewUzPz2vYyUA\npdQnwHVAD6VUPvAccJ1SKhHjMk4O8DMArfUOpdRnGDd364FHtNYN5uc8CizH6Ab6rtZ6h9O3RnR4\ndQ0OPkvP4+/f7uPIiRquGRTOExOvIN7e1erQPEdlCSx8CA58Bwl3G1W7/F3fx1y0XuOYgD98uZO9\nRyoYFOGaCmvK6ulILyYlJUWnp6dbHYZwgsbRu39dsZec0iqS+oTym0mDGd1fJhlzqtx1MP8/jLl8\npvwFRtwr0ze7idLKGkb9z7f85KoY/mtq2+orK6UytNYpLa0nI4FFu9Jas2p3MS8vN0bvDu4VzDsz\nUxg/uCdKdkzO43DAur8bRdq7xcA986BXvNVRiUvQvUsANwzpyedbCvjNpMEuGesiCUC0mw3m6N30\n3GP07R7Ea3clcsvw3jJ619mqymDRw7D3Kxh6G9z6vxAYYnVU4jKkJkezfMcRvttdzMS49p+MTxKA\ncLrtBeW8vHwP/95bQs/gAF64bRh3Xhkto3fbQ36GMaq3oggmvwwjH5JLPm7suivCCQ8OYF5GviQA\n4V72l1Ty6oq9LN1aRGiQH09PNkbvdvKX0btOpzVsfBOW/xcER8KDyyEq2eqoRBv52ny4fUQU73x/\nkJKKGsKD23ccjCQA0WaFx0/x2sp9zN+cT4CvD4+NH8hDMnq3/VSfgCWPwc5FMGgS3PZPCAqzOirh\nJKkpdt5Yc4BFWwp46Jr2Ha0tCUBcttLKGl5fvZ8P1ueChvvH9OWR6wfK6F1nq6mAYznGo+wgZLxv\nPJ/wPIx5TCp2eZiBPYNJjA5lfka+JADR8VRU1/HW2oO8s/YAp+oamJ5k5xc3xmLvFmR1aO7J4YDK\nw2d28Mdy4NjBM6+rjp69fmhfeOBL6DvWgmCFK/zu1ji6dmr/M2hJAKLVqusa+CAtl9dXZ3Osqo7J\nw3rxq4mDGNjTNYNW3FpdNRzPbX4nfywH6qvPrKt8IMQOYTEweKrRrTOsn/GzWwx06ub6+IVLJUaH\nuuR7JAGIFtU1OJiXns/fv93H4RPVjIvtwRM3XcFwu2v+SN2C1lBVeuGj+Ipzpr7y62zs1LsPhIE3\nNtnJ94Ou0TI/v3AJSQDighwOzZfbivjrir0cPHqSEX1C+eudiYwZ4KWjdxvqoDyvmR18jvGztuLs\n9YMjjR17/+vOOYrvB517SHdNYTlJAOI8WmtW7ynh5eV72Fl0gisignnr/hRuHOIFo3eryy98FF+e\nD8bUVgZbAHTra+zU+449ewcf2gf85Z6I6NgkAYizbMop489f72ZTzjH6hAXxtzsTuSWht8tL1bUb\nh8O4HNPcDv5YDpwqO3v9oO7GTt1+JcSnnr2TD46UHjjCrUkCEIAxevcv3+xh9R5j9O4fbhvGnSnR\n+Pu64Q6utsq44Vp28Pwd/PFcaKg9s66yQWi0sUMfOu3sHXy3vhAos5QKzyUJwMsdMEfvfrm1iK6d\n/Hhq8mBmusvo3eN5cCgNyg6cvZOvPHz2ev7BRo+ankPgisln7+S7RoNN/hsI7yR/+V6qqNwYvTsv\nIx9/mw+PXm+M3nVF3+PLVlMJuT9A9rewfxWU7jMXKAjpbezQB95o7Oy7mT1qusUYo2Q9/d6FEJdB\nEoCXKTtZy+vfZTNnfS5aa+4bbYzebe85Ry6LwwFHtp3Z4R9aD4468O0EMVcZ9W37XWN0pZQat0Jc\nMkkAXqKypp631x7g7bUHqaqt50cj7PzyxliiwzpYT5WKw0Yd2/2rjIpWJ0uM9ohhMPr/wYDx0GeM\n7PCFcAJJAB6uuq6BD9fn8vrq/ZSdrGVSnDF6N9ZFJedaVFdtXMff/62x4z9ilpcO6mHs7AeMhwHX\nQ3D7T40rhLdpTU3gd4GbgWKt9TCzLQz4FIjBqAl8h9b6mDI6ib8GTAGqgAe01pvN98wEnjU/9gWt\n9Wznbopoqr7BwfyMfF77dh9F5cbo3V9PvIIEFw0xvyCtoWSPucNfBTk/QP0p8PGDPqPhxt8ZO/2I\neOliKUQ7a80ZwPvA/wFzmrQ9BXyrtX5JKfWU+fpJYDIQaz5GAf8ERpkJ4zkgBaOQfIZSaonW+piz\nNkQYHA7Nsu1FvPrNXg4cPUlidCiv3JHA2AE9rAuqqsy4nLN/lXGUf6LAaO8eC8kzjR1+36sgoIt1\nMQrhhVpMAFrrNUqpmHOapwHXmc9nA6sxEsA0YI42Ks2vV0qFKqUizXVXaK3LAJRSK4BJwCdt3gIB\nGDv+f+8t4S/f7GFH4QkGRXThzfuSmTA0wvWjdxvqIH+TscPP/hYKtwDa6FPf/zoY8KRxWSe0j2vj\nEkKc5XLvAURorYsAtNZFSqmeZnsUkNdkvXyz7ULtog201uwqqmBxVgFfZBZSWF5NdFgnXr0jgWmJ\nUa4dvVt2wOyt8x0cXGPMi6NsYE+B6542jvKjksDHDcYXCOElnH0TuLk9jr5I+/kfoNQsYBZAnz5y\nhNicQ6VVLMkqYHFmIfuKK/H1UVwzKJzfTBrMlPhI14zerT5h7Oj3rzKu5x/LMdpD+0D8DGOH3+8a\n6CQzhgrRUV1uAjiilIo0j/4jgWKzPR+IbrKeHSg02687p311cx+stX4TeBMgJSWl2SThjYorqlm6\ntYjFmYVk5h0HYGRMGC/cNowp8ZGEdW7n6YMdDVCYeWaHn7fRmBjNvwvEjIMxjxo7/bD+MuhKCDdx\nuQlgCTATeMn8ubhJ+6NKqbkYN4HLzSSxHPgfpVRjJYuJwNOXH7Z3OFFdx/Lth1mSVcgP2UdxaBga\nGcJTkwdzS0JvokI7tW8A5fnmDn8VHFgNp44BCiIT4OpfGjt8+0iZu14IN9WabqCfYBy991BK5WP0\n5nkJ+Ewp9SBwCEg1V1+G0QU0G6Mb6E8AtNZlSqk/AJvM9Z5vvCEszlZd18B3u4tZnFnIqj3F1NY7\n6BMWxCPXD+TWhN7t23+/tsqYaqHx5u3RPUZ7cCRcMcXY4fe/zpjLXgjh9pTRYadjSklJ0enp6VaH\n0e7qGxykHShlcWYhy7cfpqKmnh5dArh5eCTTEnuTGB3aPj15tDYGXjXu8A+lGTNl+gYa89sPGA8D\nbjAmUZPLOkK4DaVUhtY6paX1ZCSwRbTWZOYdZ3FmIV9uLeJoZQ3BAb5MGtaLaYlRjO4fhq+tHW7m\nVhafmWph/yo4ad6+6TkURs4y++SPBb92vrwkhLCcJAAX23ekgsWZhSzJKuRQWRX+vj7cMLgn0xJ7\nc90VPQn0c3I3yfoaYxK1xpu3h7cZ7UHdof/1Z6ZbCIl07vcKITo8SQAuUHD8FF9kFbI4s5BdRSfw\nUXDVwB48Nn4gNw3rRUigE6dg1hqO7juzw8/5HuqqwMcXokfDDb81dvi9EmSqBSG8nCSAdlJ2spZl\n24pYklnIxhzjfveIPqH87pahTB3e23nTL9dVQ/EOKMqCgs1Gb51yc8xd2AAYca+xw4+5GgI6yARw\nQogOQRKAE52sqWfFziMszixg7b6j1Ds0A3t24dcTB3FrQhR9urdx6uXqE8YlnKIsOLzV+Fmy50yh\n8sBQ6DcOxj1u7PS7xbR5m4QQnksSQBvV1jtYs7eExVmFrNh5mOo6B727BvLguH5MS4hiSGTw5fXg\nqSyBw1nGTr7I3NkfO3hmeZdeEDkcBk+FXsON56F9pbeOEKLVJAFcBodDs+FgGUuyCli27TDlp+ro\nFuTHjGQ70xKjSO7TDZ/WzsOjtXHJpnEnf3ir8byi8Mw63WKMnfyIeyAy0XgeHNEu2yaE8B6SAFpJ\na82OwhMszizgi6wiDp+oJsjfxsShEUxLjOLq2B74tdRt09EApfvNnXymsaM/vNUcYQsoH+gxyLiM\n02u4MeK2V7zMpyOEaBeSAFpw8OhJlmQWsjirgAMlJ/GzKa4d1JP/mjqEG4b0JMj/Ar/C+loo2XX2\nkf3h7VB30lhu8zf63g+51bh80ysBIuLAv4OVaBRCeCxJAM04cqKaL7IK+SKrkKz8cpSCUf3CeGhc\nfyYP60Vo0Dlz39SeNHbujTdmi7KgeJdRwByMCdN6xRs9ciITjB1++GCwObH7pxBCXCJJAKbyqjq+\n3mHMtpl2oBStIT6qK89OHcLNw3vTq6tZhLyqDA5sPXP5pijL6HffOLt1pzBjJz/mYfMSToIxQ6b0\nuRdCdDBenQCq6xr4dlcxizMLWL2nhNoGB/16dObn42O5NSGSAYGVxg4+c/6Z3jjlh858QEiUsZOP\nu/3MkX1IlPTEEUK4Ba9LAPUNDr7PPsqSzEKW7zjMydoGenbx5+cjbNwSUUafmnTU4a0wOwtOlpx5\nY9gAsCfDlf9x5gatzIophHBjXpEAtNZsPnSMxZmFfJ2VT+ipHFIC8ngj/AjxtkOElO9CbT8B2zGm\nTAgfDAMnnDmqjxgGgSFWb4YQQjiVRyeAvfklrF+/lsN7NhJ1ai/TbTk865OHf0CtsUJ5oLFzj59h\nXq8fbvTM8Qu0NnAhhHABj0wAhXn7qXnvdvo3HGKQcgBQGxiMT+8EfHtPMo7qIxOgeyzYPPJXIIQQ\nLfLIvV/PCDs7AiLYFXkjMXFjCO6XjH+3GLk5K4QQTXhkAvD1DyDhyW+sDkMIITq0NnVOV0rlKKW2\nKaUylVLpZluYUmqFUmqf+bOb2a6UUn9XSmUrpbYqpZKcsQFCCCEujzNGJ12vtU5sUn/yKeBbrXUs\n8K35GmAyEGs+ZgH/dMJ3CyGEuEztMTx1GjDbfD4buK1J+xxtWA+EKqWkDqEQQlikrQlAA98opTKU\nUrPMtgitdRGA+bOn2R4F5DV5b77Zdhal1CylVLpSKr2kpOTcxUIIIZykrTeBr9JaFyqlegIrlFK7\nL7Juc11w9HkNWr8JvAmQkpJy3nIhhBDO0aYzAK11ofmzGPgcGAkcaby0Y/4sNlfPB6KbvN0ONKl6\nIoQQwpUuOwEopTorpYIbnwMTMSZTWALMNFebCSw2ny8B7jd7A40GyhsvFQkhhHC9tlwCigA+N+vd\n+gIfa62/VkptAj5TSj0IHAJSzfWXAVOAbKAK+EkbvlsIIUQbKa077mV2pVQJkNuGj+gBHHVSOO7C\n27bZ27YXZJu9RVu2ua/WOryllTp0AmgrpVR6k/EJXsHbttnbthdkm72FK7ZZylQJIYSXkgQghBBe\nytMTwJtWB2ABb9tmb9tekG32Fu2+zR59D0AIIcSFefoZgBBCiAvwyASglJqklNpjTj39VMvvcG9K\nqXeVUsVKqe1Wx+IqSqlopdR3SqldSqkdSqlfWB1Te1NKBSqlNiqlssxt/r3VMbmCUsqmlNqilPrS\n6lhcpbmp9tvlezztEpBSygbsBSZgTD+xCbhba73T0sDakVLqGqASY7bVYVbH4wrmNCORWuvN5oj0\nDOA2D/93VkBnrXWlUsoP+B74hTm7rsdSSj0OpAAhWuubrY7HFZRSOUCK1rpdxz544hnASCBba31A\na10LzMWYitpjaa3XAGVWx+FKWusirfVm83kFsItmZpf1JOZU6pXmSz/z4VlHcOdQStmBqcDbVsfi\niTwxAbRq2mnhOZRSMcAIYIO1kbQ/83JIJsYkiyu01p6+zX8DfgM4rA7ExZqbat/pPDEBtGraaeEZ\nlFJdgAXAL7XWJ6yOp71prRu01okYs+mOVEp57CU/pdTNQLHWOsPqWCxwldY6CaOS4iPmZV6n88QE\nINNOewnzOvgC4COt9UKr43ElrfVxYDUwyeJQ2tNVwK3m9fC5wHil1IfWhuQaF5hq3+k8MQFsAmKV\nUv2UUv7AXRhTUQsPYt4QfQfYpbV+1ep4XEEpFa6UCjWfdwJuBC5WhMmtaa2f1lrbtdYxGP+PV2mt\n77U4rHZ3kan2nc7jEoDWuh54FFiOcWPwM631Dmujal9KqU+ANOAKpVS+ORW3p7sKuA/jqDDTfEyx\nOqh2Fgl8p5TainGgs0Jr7TVdI71IBPC9UioL2Ags1Vp/3R5f5HHdQIUQQrSOx50BCCGEaB1JAEII\n4aUkAQghhJeSBCCEEF5KEoAQQngpSQBCCOGlJAEIIYSXkgQghBBe6v8Hx0u6aIFv9tIAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f14cbf28860>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▌         | 584/10000 [02:24<38:50,  4.04it/s]"
     ]
    }
   ],
   "source": [
    "for i in trange(5000):",
    "\n",
    "    sess.run(train_step, sample_batch())",
    "\n",
    "\n",
    "    if i % 100 == 0:",
    "\n",
    "        rewards_history.append(np.mean(evaluate(agent, env, n_games=1)))",
    "\n",
    "        clear_output(True)",
    "\n",
    "        plt.plot(rewards_history, label='rewards')",
    "\n",
    "        plt.plot(moving_average(np.array(rewards_history),",
    "\n",
    "                                span=10), label='rewards ewma@10')",
    "\n",
    "        plt.legend()",
    "\n",
    "        plt.show()",
    "\n",
    "        if rewards_history[-1] >= 10000:",
    "\n",
    "            print(\"Your agent has just passed the minimum homework threshold\")",
    "\n",
    "            break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### \"Final\" evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_monitor = gym.wrappers.Monitor(env, directory=\"kungfu_videos\", force=True)",
    "\n",
    "final_rewards = evaluate(agent, env_monitor, n_games=20,)",
    "\n",
    "env_monitor.close()",
    "\n",
    "print(\"Final mean reward\": np.mean(final_rewards))",
    "\n",
    "\n",
    "video_names = list(filter(lambda s: s.endswith(",
    "\n",
    "    \".mp4\"), os.listdir(\"./kungfu_videos/\")))",
    "\n",
    "HTML(\"\"\"",
    "\n",
    "<video width=\"640\" height=\"480\" controls>",
    "\n",
    "  <source src=\"{}\" type=\"video/mp4\">",
    "\n",
    "</video>",
    "\n",
    "\"\"\".format(\"./kungfu_videos/\"+video_names[-1]))  # this may or may not be _last_ video. Try other indices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### POMDP setting\n",
    "\n",
    "The atari game we're working with is actually a POMDP: your agent needs to know timing at which enemies spawn and move, but cannot do so unless it has some memory. \n",
    "\n",
    "Let's design another agent that has a recurrent neural net memory to solve this.\n",
    "\n",
    "__Note:__ it's also a good idea to scale rollout_len up to learn longer sequences. You may wish set it to >=20 or to start at 10 and then scale up as time passes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleRecurrentAgent:",
    "\n",
    "    def __init__(self, name, obs_shape, n_actions, reuse=False):",
    "\n",
    "        \"\"\"A simple actor-critic agent\"\"\"",
    "\n",
    "\n",
    "        with tf.variable_scope(name, reuse=reuse):",
    "\n",
    "            # Note: number of units/filters is arbitrary, you can and should change it at your will",
    "\n",
    "            self.conv0 = Conv2D(32, (3, 3), strides=(2, 2), activation='elu')",
    "\n",
    "            self.conv1 = Conv2D(32, (3, 3), strides=(2, 2), activation='elu')",
    "\n",
    "            self.conv2 = Conv2D(32, (3, 3), strides=(2, 2), activation='elu')",
    "\n",
    "            self.flatten = Flatten()",
    "\n",
    "            self.hid = Dense(128, activation='elu')",
    "\n",
    "\n",
    "            self.rnn0 = tf.nn.rnn_cell.GRUCell(256, activation=tf.tanh)",
    "\n",
    "\n",
    "            self.logits = Dense(n_actions)",
    "\n",
    "            self.state_value = Dense(1)",
    "\n",
    "\n",
    "            # prepare a graph for agent step",
    "\n",
    "            _initial_state = self.get_initial_state(1)",
    "\n",
    "            self.prev_state_placeholders = [tf.placeholder(m.dtype,",
    "\n",
    "                                                           [None] + [m.shape[i] for i in range(1, m.ndim)])",
    "\n",
    "                                            for m in _initial_state]",
    "\n",
    "            self.obs_t = tf.placeholder('float32', [None, ] + list(obs_shape))",
    "\n",
    "            self.next_state, self.agent_outputs = self.symbolic_step(",
    "\n",
    "                self.prev_state_placeholders, self.obs_t)",
    "\n",
    "\n",
    "    def symbolic_step(self, prev_state, obs_t):",
    "\n",
    "        \"\"\"Takes agent's previous step and observation, returns next state and whatever it needs to learn (tf tensors)\"\"\"",
    "\n",
    "\n",
    "        nn = self.conv0(obs_t)",
    "\n",
    "        nn = self.conv1(nn)",
    "\n",
    "        nn = self.conv2(nn)",
    "\n",
    "        nn = self.flatten(nn)",
    "\n",
    "        nn = self.hid(nn)",
    "\n",
    "\n",
    "        (prev_rnn0,) = prev_state",
    "\n",
    "\n",
    "        # YOUR CODE: apply recurrent neural net for one step here.",
    "\n",
    "        # See docs on self.rnn0(...)",
    "\n",
    "        # the recurrent cell should take the last feedforward dense layer as input",
    "\n",
    "        raise NotImplementedError(\"Please implement rnn step\")",
    "\n",
    "\n",
    "        logits = self.logits( < some layer > )",
    "\n",
    "        state_value = self.state_value( < some layer > )",
    "\n",
    "\n",
    "        new_state = [new_rnn0]",
    "\n",
    "\n",
    "        return new_state, (logits, state_value)",
    "\n",
    "\n",
    "    def get_initial_state(self, batch_size):",
    "\n",
    "        \"\"\"Return a list of agent memory states at game start. Each state is a np array of shape [batch_size, ...]\"\"\"",
    "\n",
    "        # feedforward agent has no state",
    "\n",
    "        return [np.zeros([batch_size, self.rnn0.output_size], 'float32')]",
    "\n",
    "\n",
    "    def step(self, prev_state, obs_t):",
    "\n",
    "        \"\"\"Same as symbolic state except it operates on numpy arrays\"\"\"",
    "\n",
    "        sess = tf.get_default_session()",
    "\n",
    "        feed_dict = {self.obs_t: obs_t}",
    "\n",
    "        for state_ph, state_value in zip(self.prev_state_placeholders, prev_state):",
    "\n",
    "            feed_dict[state_ph] = state_value",
    "\n",
    "        return sess.run([self.next_state, self.agent_outputs], feed_dict)",
    "\n",
    "\n",
    "    def sample_actions(self, agent_outputs):",
    "\n",
    "        \"\"\"pick actions given numeric agent outputs (np arrays)\"\"\"",
    "\n",
    "        logits, state_values = agent_outputs",
    "\n",
    "        policy = np.exp(logits) / np.sum(np.exp(logits),",
    "\n",
    "                                         axis=-1, keepdims=True)",
    "\n",
    "        return [np.random.choice(len(p), p=p) for p in policy]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = SimpleRecurrentAgent('agent_with_memory', obs_shape, n_actions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now let's train it!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A whole lot of your code here: train the new agent with GRU memory.",
    "\n",
    "# - create pool",
    "\n",
    "# - write loss functions and training op",
    "\n",
    "# - train",
    "\n",
    "# You can reuse most of the code with zero to few changes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "\n",
    "```\n",
    "```\n",
    "\n",
    "```\n",
    "```\n",
    "\n",
    "```\n",
    "```\n",
    "\n",
    "```\n",
    "```\n",
    "\n",
    "```\n",
    "```\n",
    "\n",
    "```\n",
    "```\n",
    "\n",
    "```\n",
    "```\n",
    "\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Homework assignment is in the second notebook: [url]"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
